piral-cli 0.15.0-alpha.3905 → 0.15.0-alpha.4005

package/src/apps/debug-pilet.ts

@@ -16,6 +16,8 @@ import {
   fail,
   log,
   logDone,
+  cpuCount,
+  concurrentWorkers,
 } from '../common';
 
 export interface DebugPiletOptions {
@@ -57,6 +59,11 @@ export interface DebugPiletOptions {
    */
   port?: number;
 
+  /**
+   * Sets the maximum number of parallel build processes.
+   */
+  concurrency?: number;
+
   /**
    * Defines if hot module reloading (HMR) should be integrated for faster debugging.
    */
@@ -112,6 +119,7 @@ export const debugPiletDefaults: DebugPiletOptions = {
   hmr: true,
   optimizeModules: false,
   schemaVersion: config.schemaVersion,
+  concurrency: cpuCount,
 };
 
 const injectorName = resolve(__dirname, '../injectors/pilet.js');
@@ -168,6 +176,7 @@ export async function debugPilet(baseDir = process.cwd(), options: DebugPiletOpt
     open = debugPiletDefaults.open,
     hmr = debugPiletDefaults.hmr,
     logLevel = debugPiletDefaults.logLevel,
+    concurrency = debugPiletDefaults.concurrency,
     optimizeModules = debugPiletDefaults.optimizeModules,
     schemaVersion = debugPiletDefaults.schemaVersion,
     _ = {},
@@ -204,63 +213,61 @@ export async function debugPilet(baseDir = process.cwd(), options: DebugPiletOpt
   process.stdout.setMaxListeners(maxListeners);
   process.stdin.setMaxListeners(maxListeners);
 
-  const pilets = await Promise.all(
-    allEntries.map(async (entryModule) => {
-      const targetDir = dirname(entryModule);
-      const { peerDependencies, peerModules, root, appPackage, appFile, ignored, emulator, importmap } =
-        await retrievePiletData(targetDir, app);
-      const externals = [...Object.keys(peerDependencies), ...peerModules];
-      const mocks = join(targetDir, 'mocks');
-      const dest = resolve(root, target);
-      const outDir = dirname(dest);
-      const outFile = basename(dest);
-      const exists = await checkExistingDirectory(mocks);
-
-      if (exists) {
-        if (krasConfig.directory === undefined) {
-          krasConfig.directory = mocks;
-        }
-
-        krasConfig.sources.push(mocks);
+  const pilets = await concurrentWorkers(allEntries, concurrency, async (entryModule) => {
+    const targetDir = dirname(entryModule);
+    const { peerDependencies, peerModules, root, appPackage, appFile, ignored, emulator, importmap } =
+      await retrievePiletData(targetDir, app);
+    const externals = [...Object.keys(peerDependencies), ...peerModules];
+    const mocks = join(targetDir, 'mocks');
+    const dest = resolve(root, target);
+    const outDir = dirname(dest);
+    const outFile = basename(dest);
+    const exists = await checkExistingDirectory(mocks);
+
+    if (exists) {
+      if (krasConfig.directory === undefined) {
+        krasConfig.directory = mocks;
       }
 
-      await hooks.beforeBuild?.({ root, importmap, entryModule, schemaVersion });
-
-      const bundler = await callPiletDebug(
-        {
-          root,
-          piral: appPackage.name,
-          optimizeModules,
-          hmr,
-          externals,
-          targetDir,
-          importmap,
-          outFile,
-          outDir,
-          entryModule: `./${relative(root, entryModule)}`,
-          logLevel,
-          version: schemaVersion,
-          ignored,
-          _,
-        },
-        bundlerName,
-      );
-
-      bundler.on((args) => {
-        hooks.afterBuild?.({ ...args, root, importmap, entryModule, schemaVersion, bundler, outFile, outDir });
-      });
-
-      return {
-        emulator,
-        appFile,
-        appVersion: appPackage.version,
-        externals,
-        piral: appPackage.name,
-        bundler,
+      krasConfig.sources.push(mocks);
+    }
+
+    await hooks.beforeBuild?.({ root, importmap, entryModule, schemaVersion });
+
+    const bundler = await callPiletDebug(
+      {
         root,
-      };
-    }),
-  );
+        piral: appPackage.name,
+        optimizeModules,
+        hmr,
+        externals,
+        targetDir,
+        importmap,
+        outFile,
+        outDir,
+        entryModule: `./${relative(root, entryModule)}`,
+        logLevel,
+        version: schemaVersion,
+        ignored,
+        _,
+      },
+      bundlerName,
+    );
+
+    bundler.on((args) => {
+      hooks.afterBuild?.({ ...args, root, importmap, entryModule, schemaVersion, bundler, outFile, outDir });
+    });
+
+    return {
+      emulator,
+      appFile,
+      appVersion: appPackage.version,
+      externals,
+      piral: appPackage.name,
+      bundler,
+      root,
+    };
+  });
 
   // sanity check see #250
   checkSanity(pilets);

package/src/apps/publish-pilet.ts

@@ -1,6 +1,6 @@
 import { relative, dirname, basename, resolve } from 'path';
 import { callPiletBuild } from '../bundler';
-import { LogLevels, PiletSchemaVersion, PiletPublishSource } from '../types';
+import { LogLevels, PiletSchemaVersion, PiletPublishSource, PiletPublishScheme } from '../types';
 import {
   postFile,
   readBinary,
@@ -72,11 +72,21 @@ export interface PublishPiletOptions {
    */
   fields?: Record<string, string>;
 
+  /**
+   * Places additional headers that should be posted to the feed service.
+   */
+  headers?: Record<string, string>;
+
   /**
    * Sets the bundler to use for building, if any specific.
    */
   bundlerName?: string;
 
+  /**
+   * Sets the authorization scheme to use.
+   */
+  mode?: PiletPublishScheme;
+
   /**
    * Additional arguments for a specific bundler.
    */
@@ -90,8 +100,10 @@ export const publishPiletDefaults: PublishPiletOptions = {
   cert: undefined,
   logLevel: LogLevels.info,
   schemaVersion: config.schemaVersion,
+  mode: 'basic',
   from: 'local',
   fields: {},
+  headers: {},
 };
 
 async function getFiles(
@@ -200,6 +212,8 @@ export async function publishPilet(baseDir = process.cwd(), options: PublishPile
     schemaVersion = publishPiletDefaults.schemaVersion,
     cert = config.cert ?? publishPiletDefaults.cert,
     fields = publishPiletDefaults.fields,
+    headers = publishPiletDefaults.headers,
+    mode = publishPiletDefaults.mode,
     _ = {},
     bundlerName,
   } = options;
@@ -240,7 +254,7 @@ export async function publishPilet(baseDir = process.cwd(), options: PublishPile
 
     if (content) {
       progress(`Publishing "%s" ...`, file, url);
-      const result = await postFile(url, apiKey, content, fields, ca);
+      const result = await postFile(url, mode, apiKey, content, fields, headers, ca);
 
       if (result.success) {
         successfulUploads.push(file);

package/src/commands.ts

@@ -15,6 +15,7 @@ import {
   fromKeys,
   bundlerKeys,
   piralBuildTypeKeys,
+  publishModeKeys,
 } from './helpers';
 import {
   ToolCommand,
@@ -25,6 +26,7 @@ import {
   PiletPublishSource,
   PiletSchemaVersion,
   PiletBuildType,
+  PiletPublishScheme,
 } from './types';
 
 function specializeCommand(commands: Array<ToolCommand<any>>, command: ToolCommand<any>, suffix: string) {
@@ -414,6 +416,9 @@ const allCommands: Array<ToolCommand<any>> = [
         .number('log-level')
         .describe('log-level', 'Sets the log level to use (1-5).')
         .default('log-level', apps.debugPiletDefaults.logLevel)
+        .number('concurrency')
+        .describe('concurrency', 'Sets the maximum number of concurrent build jobs.')
+        .default('concurrency', apps.debugPiletDefaults.concurrency)
         .boolean('open')
         .describe('open', 'Opens the pilet directly in the browser.')
         .default('open', apps.debugPiletDefaults.open)
@@ -452,6 +457,7 @@ const allCommands: Array<ToolCommand<any>> = [
         logLevel: args['log-level'] as LogLevels,
         open: args.open as boolean,
         schemaVersion: args.schema as PiletSchemaVersion,
+        concurrency: args.concurrency as number,
         feed: args.feed as string,
         hooks: args.hooks as object,
         _: args,
@@ -477,6 +483,9 @@ const allCommands: Array<ToolCommand<any>> = [
         .number('log-level')
         .describe('log-level', 'Sets the log level to use (1-5).')
         .default('log-level', apps.buildPiletDefaults.logLevel)
+        .number('concurrency')
+        .describe('concurrency', 'Sets the maximum number of concurrent build jobs.')
+        .default('concurrency', apps.buildPiletDefaults.concurrency)
         .boolean('source-maps')
         .describe('source-maps', 'Creates source maps for the bundles.')
         .default('source-maps', apps.buildPiletDefaults.sourceMaps)
@@ -524,6 +533,7 @@ const allCommands: Array<ToolCommand<any>> = [
         fresh: args.fresh as boolean,
         logLevel: args['log-level'] as LogLevels,
         schemaVersion: args.schema as PiletSchemaVersion,
+        concurrency: args.concurrency as number,
         app: args.app as string,
         hooks: args.hooks as object,
         _: args,
@@ -590,6 +600,9 @@ const allCommands: Array<ToolCommand<any>> = [
         .choices('schema', schemaKeys)
         .describe('schema', 'Sets the schema to be used when making a fresh build of the pilet.')
         .default('schema', apps.publishPiletDefaults.schemaVersion)
+        .choices('mode', publishModeKeys)
+        .describe('mode', 'Sets the authorization mode to use.')
+        .default('mode', apps.publishPiletDefaults.mode)
         .choices('bundler', availableBundlers)
         .describe('bundler', 'Sets the bundler to use.')
         .default('bundler', availableBundlers[0])
@@ -599,6 +612,9 @@ const allCommands: Array<ToolCommand<any>> = [
         .option('fields', undefined)
         .describe('fields', 'Sets additional fields to be included in the feed service request.')
         .default('fields', apps.publishPiletDefaults.fields)
+        .option('headers', undefined)
+        .describe('headers', 'Sets additional headers to be included in the feed service request.')
+        .default('headers', apps.publishPiletDefaults.headers)
         .string('base')
         .default('base', process.cwd())
         .describe('base', 'Sets the base directory. By default the current directory is used.');
@@ -615,6 +631,8 @@ const allCommands: Array<ToolCommand<any>> = [
         from: args.from as PiletPublishSource,
         schemaVersion: args.schema as PiletSchemaVersion,
         fields: args.fields as Record<string, string>,
+        headers: args.headers as Record<string, string>,
+        mode: args.mode as PiletPublishScheme,
         _: args,
       });
     },

package/src/common/http.test.ts

@@ -71,7 +71,7 @@ jest.mock('axios', () => ({
 
 describe('HTTP Module', () => {
   it('postFile form posts a file successfully should be ok', async () => {
-    const result = await postFile(apiUrl, '123', Buffer.from('example'));
+    const result = await postFile(apiUrl, 'basic', '123', Buffer.from('example'));
     expect(result).toEqual({
       response: undefined,
       status: 200,
@@ -80,7 +80,7 @@ describe('HTTP Module', () => {
   });
 
   it('postFile form fails to post file should be false', async () => {
-    const result = await postFile(apiUrl, '124', Buffer.from('example'));
+    const result = await postFile(apiUrl, 'basic', '124', Buffer.from('example'));
     expect(result).toEqual({
       response: '',
       status: 401,
@@ -89,7 +89,7 @@ describe('HTTP Module', () => {
   });
 
   it('postFile form not found to post file should be false', async () => {
-    const result = await postFile('http://sample.com/', '', Buffer.from('example'));
+    const result = await postFile('http://sample.com/', 'basic', '', Buffer.from('example'));
     expect(result).toEqual({
       response: '',
       status: 404,
@@ -99,7 +99,7 @@ describe('HTTP Module', () => {
 
   it('postFile call results in error request', async () => {
     errorRequest = true;
-    const result = await postFile('http://sample.com/', '', Buffer.from('example'));
+    const result = await postFile('http://sample.com/', 'basic', '', Buffer.from('example'));
     expect(result).toEqual({
       response: undefined,
       status: 500,
@@ -110,7 +110,7 @@ describe('HTTP Module', () => {
 
   it('postFile call results in error other', async () => {
     errorOther = true;
-    const result = await postFile('http://sample.com/', '', Buffer.from('example'));
+    const result = await postFile('http://sample.com/', 'basic', '', Buffer.from('example'));
     expect(result).toEqual({
       response: undefined,
       status: 500,
@@ -121,7 +121,7 @@ describe('HTTP Module', () => {
 
   it('postFile call results in error response', async () => {
     errorResponse = true;
-    let result = await postFile('http://sample.com/', '', Buffer.from('example'));
+    let result = await postFile('http://sample.com/', 'basic', '', Buffer.from('example'));
     expect(result).toEqual({
       response: 'This component is not available anymore.',
       status: 410,
@@ -129,7 +129,7 @@ describe('HTTP Module', () => {
     });
     errorResponse = false;
     errorResponse2 = true;
-    result = await postFile('http://sample.com/', '', Buffer.from('example'));
+    result = await postFile('http://sample.com/', 'basic', '', Buffer.from('example'));
     expect(result).toEqual({
       response: 'This component is not available anymore.',
       status: 410,

package/src/common/http.ts

@@ -5,6 +5,7 @@ import { platform, tmpdir } from 'os';
 import { createWriteStream } from 'fs';
 import { log } from './log';
 import { axios, FormData } from '../external';
+import { PiletPublishScheme } from '../types';
 
 const os = platform();
 const standardHeaders = {
@@ -67,25 +68,42 @@ export interface PostFileResult {
 
 export function postFile(
   target: string,
+  scheme: PiletPublishScheme,
   key: string,
   file: Buffer,
-  fields: Record<string, string> = {},
+  customFields: Record<string, string> = {},
+  customHeaders: Record<string, string> = {},
   ca?: Buffer,
 ): Promise<PostFileResult> {
   const form = new FormData();
   const httpsAgent = ca ? new Agent({ ca }) : undefined;
 
-  Object.keys(fields).forEach((key) => form.append(key, fields[key]));
+  Object.keys(customFields).forEach((key) => form.append(key, customFields[key]));
 
   form.append('file', file, 'pilet.tgz');
 
   const headers: Record<string, string> = {
     ...form.getHeaders(),
     ...standardHeaders,
+    ...customHeaders,
   };
 
   if (key) {
-    headers.authorization = `Basic ${key}`;
+    switch (scheme) {
+      case 'basic':
+        headers.authorization = `Basic ${key}`;
+        break;
+      case 'bearer':
+        headers.authorization = `Bearer ${key}`;
+        break;
+      case 'digest':
+        headers.authorization = `Digest ${key}`;
+        break;
+      case 'none':
+      default:
+        headers.authorization = key;
+        break;
+    }
   }
 
   return axios.default

package/src/common/index.ts

@@ -20,6 +20,7 @@ export * from './merge';
 export * from './npm';
 export * from './pack';
 export * from './package';
+export * from './parallel';
 export * from './patcher';
 export * from './patches';
 export * from './port';

package/src/common/info.ts

@@ -1,3 +1,5 @@
+import { cpus } from 'os';
+
 const info = require('../../package.json');
 
 export function findCompatVersion(version: string) {
@@ -18,3 +20,4 @@ export const compatVersion = findCompatVersion(cliVersion);
 export const repositoryUrl = info.repository.url;
 export const isWindows = process.platform === 'win32';
 export const pathSeparator = isWindows ? ';' : ':';
+export const cpuCount = cpus().length;

package/src/common/parallel.test.ts

@@ -0,0 +1,28 @@
+import { concurrentWorkers } from './parallel';
+
+describe('Concurrent Workers', () => {
+  it('Can run against no entries', async () => {
+    const result = await concurrentWorkers([], 10, () => Promise.resolve('foo'));
+    expect(result).toEqual([]);
+  });
+
+  it('Can run against single entry', async () => {
+    const result = await concurrentWorkers(['bar'], 10, (item) => Promise.resolve('foo' + item));
+    expect(result).toEqual(['foobar']);
+  });
+
+  it('Can run against less entries than concurrency', async () => {
+    const result = await concurrentWorkers(['bar', 'rba', 'abr', 'rab', 'arb', 'bra'], 10, (item) => Promise.resolve('foo' + item));
+    expect(result).toEqual(['foobar', 'foorba', 'fooabr', 'foorab', 'fooarb', 'foobra']);
+  });
+
+  it('Can run against more entries than concurrency', async () => {
+    const result = await concurrentWorkers(['bar', 'rba', 'abr', 'rab', 'arb', 'bra'], 2, (item) => Promise.resolve('foo' + item));
+    expect(result).toEqual(['foobar', 'foorba', 'fooabr', 'foorab', 'fooarb', 'foobra']);
+  });
+
+  it('Can run against equal entries than concurrency', async () => {
+    const result = await concurrentWorkers(['bar', 'rba', 'abr', 'rab', 'arb', 'bra'], 6, (item) => Promise.resolve('foo' + item));
+    expect(result).toEqual(['foobar', 'foorba', 'fooabr', 'foorab', 'fooarb', 'foobra']);
+  });
+});

package/src/common/parallel.ts

@@ -0,0 +1,21 @@
+export async function concurrentWorkers<T, R>(
+  items: Array<T>,
+  concurrency: number,
+  worker: (item: T) => Promise<R>,
+): Promise<Array<R>> {
+  const maxItems = items.length;
+  const results: Array<R> = new Array(maxItems);
+  let offset = 0;
+
+  await Promise.all(
+    items.slice(0, concurrency).map(async () => {
+      while (offset < maxItems) {
+        const i = offset++;
+        const item = items[i];
+        results[i] = await worker(item);
+      }
+    }),
+  );
+
+  return results;
+}

package/src/helpers.ts

@@ -7,9 +7,11 @@ import {
   PiletPublishSource,
   PiralBuildType,
   PiletBuildType,
+  PiletPublishScheme,
 } from './types';
 
 export const schemaKeys: Array<PiletSchemaVersion> = ['v0', 'v1', 'v2', 'none'];
+export const publishModeKeys: Array<PiletPublishScheme> = ['none', 'basic', 'bearer', 'digest'];
 export const fromKeys: Array<PiletPublishSource> = ['local', 'remote', 'npm'];
 export const piralBuildTypeKeys: Array<PiralBuildType> = ['all', 'release', 'emulator', 'emulator-sources'];
 export const piletBuildTypeKeys: Array<PiletBuildType> = ['default', 'standalone', 'manifest'];

package/src/types/public.ts

@@ -218,6 +218,8 @@ export interface BundlerDefinition {
 
 export type PiletSchemaVersion = 'none' | 'v0' | 'v1' | 'v2';
 
+export type PiletPublishScheme = 'none' | 'digest' | 'bearer' | 'basic';
+
 export type PiletPublishSource = 'local' | 'npm' | 'remote';
 
 export type PiralBuildType = 'all' | 'release' | 'emulator' | 'emulator-sources';