sharetribe-cli 1.15.1 → 1.16.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sharetribe-cli",
-  "version": "1.15.1",
+  "version": "1.16.0",
   "description": "Unofficial Sharetribe CLI - 100% compatible with flex-cli",
   "type": "module",
   "main": "./dist/index.js",
@@ -41,12 +41,14 @@
     "commander": "^12.1.0",
     "inquirer": "^9.2.23",
     "jsedn": "^0.4.1",
-    "sharetribe-flex-build-sdk": "^1.15.1",
-    "yargs": "^18.0.0"
+    "sharetribe-flex-build-sdk": "^1.15.2",
+    "yargs": "^18.0.0",
+    "yauzl": "^3.2.0"
   },
   "devDependencies": {
     "@types/inquirer": "^9.0.7",
     "@types/node": "^20.17.10",
+    "@types/yauzl": "^2.10.3",
     "@typescript-eslint/eslint-plugin": "^8.18.2",
     "@typescript-eslint/parser": "^8.18.2",
     "esbuild": "^0.27.2",

package/src/commands/assets/index.ts

@@ -4,15 +4,31 @@
 
 import { Command } from 'commander';
 import {
-  pullAssets as sdkPullAssets,
   pushAssets as sdkPushAssets,
+  stageAsset as sdkStageAsset,
+  getApiBaseUrl,
+  readAuth,
 } from 'sharetribe-flex-build-sdk';
 import { printError } from '../../util/output.js';
-import { readFileSync, writeFileSync, existsSync, mkdirSync, readdirSync, statSync, unlinkSync } from 'node:fs';
+import {
+  readFileSync,
+  writeFileSync,
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  statSync,
+  unlinkSync,
+  createWriteStream,
+} from 'node:fs';
 import { join, dirname } from 'node:path';
 import { createHash } from 'node:crypto';
+import * as http from 'node:http';
+import * as https from 'node:https';
+import { tmpdir } from 'node:os';
+import { pipeline } from 'node:stream/promises';
 import chalk from 'chalk';
 import edn from 'jsedn';
+import yauzl from 'yauzl';
 
 
 interface AssetMetadata {
@@ -20,21 +36,15 @@ interface AssetMetadata {
   assets: Array<{ path: string; 'content-hash': string }>;
 }
 
-/**
- * Reads asset metadata from .flex-cli/asset-meta.edn
- */
-function readAssetMetadata(basePath: string): AssetMetadata | null {
-  const metaPath = join(basePath, '.flex-cli', 'asset-meta.edn');
-  if (!existsSync(metaPath)) {
-    return null;
-  }
+const ASSET_META_FILENAME = 'meta/asset-meta.edn';
+const ASSETS_DIR = 'assets/';
+const CLEAR_LINE = '\x1b[K';
+const CARRIAGE_RETURN = '\r';
 
+function parseAssetMetadataEdn(content: string): AssetMetadata | null {
   try {
-    const content = readFileSync(metaPath, 'utf-8');
     const parsed = edn.parse(content);
-
-    // Convert EDN map to JavaScript object
-    const version = parsed.at(edn.kw(':version'));
+    const version = parsed.at(edn.kw(':version')) || parsed.at(edn.kw(':aliased-version'));
     const assets = parsed.at(edn.kw(':assets'));
 
     const assetList: Array<{ path: string; 'content-hash': string }> = [];
@@ -47,12 +57,33 @@ function readAssetMetadata(basePath: string): AssetMetadata | null {
       }
     }
 
+    if (!version) {
+      return null;
+    }
+
     return { version, assets: assetList };
   } catch {
     return null;
   }
 }
 
+/**
+ * Reads asset metadata from .flex-cli/asset-meta.edn
+ */
+function readAssetMetadata(basePath: string): AssetMetadata | null {
+  const metaPath = join(basePath, '.flex-cli', 'asset-meta.edn');
+  if (!existsSync(metaPath)) {
+    return null;
+  }
+
+  try {
+    const content = readFileSync(metaPath, 'utf-8');
+    return parseAssetMetadataEdn(content);
+  } catch {
+    return null;
+  }
+}
+
 /**
  * Writes asset metadata to .flex-cli/asset-meta.edn
  */
@@ -79,7 +110,8 @@ function writeAssetMetadata(basePath: string, metadata: AssetMetadata): void {
 }
 
 /**
- * Calculates SHA-1 hash of file content
+ * Calculates SHA-1 hash of file content matching backend convention
+ * Content is prefixed with `${byte-count}|` before hashing
  */
 function calculateHash(data: Buffer): string {
   const prefix = Buffer.from(`${data.length}|`, 'utf-8');
@@ -97,6 +129,7 @@ function readLocalAssets(basePath: string): Array<{ path: string; data: Buffer;
 
     for (const entry of entries) {
       if (entry === '.flex-cli') continue; // Skip metadata directory
+      if (entry === '.DS_Store') continue; // Skip .DS_Store files
 
       const fullPath = join(dir, entry);
       const relPath = relativePath ? join(relativePath, entry) : entry;
@@ -116,6 +149,35 @@ function readLocalAssets(basePath: string): Array<{ path: string; data: Buffer;
   return assets;
 }
 
+/**
+ * Lists local asset paths without reading file data
+ */
+function listLocalAssetPaths(basePath: string): string[] {
+  const assets: string[] = [];
+
+  function scanDir(dir: string, relativePath: string = '') {
+    const entries = readdirSync(dir);
+
+    for (const entry of entries) {
+      if (entry === '.flex-cli') continue;
+      if (entry === '.DS_Store') continue;
+
+      const fullPath = join(dir, entry);
+      const relPath = relativePath ? join(relativePath, entry) : entry;
+      const stat = statSync(fullPath);
+
+      if (stat.isDirectory()) {
+        scanDir(fullPath, relPath);
+      } else if (stat.isFile()) {
+        assets.push(relPath);
+      }
+    }
+  }
+
+  scanDir(basePath);
+  return assets;
+}
+
 /**
  * Validates JSON files
  */
@@ -131,6 +193,186 @@ function validateJsonAssets(assets: Array<{ path: string; data: Buffer }>): void
   }
 }
 
+function formatDownloadProgress(bytes: number): string {
+  const mb = bytes / 1024 / 1024;
+  return `${CARRIAGE_RETURN}${CLEAR_LINE}Downloaded ${mb.toFixed(2)}MB`;
+}
+
+function printDownloadProgress(stream: NodeJS.ReadableStream): void {
+  let downloaded = 0;
+  const printProgress = (): void => {
+    process.stderr.write(formatDownloadProgress(downloaded));
+  };
+  const interval = setInterval(printProgress, 100);
+
+  stream.on('data', (chunk: Buffer) => {
+    downloaded += chunk.length;
+  });
+
+  stream.on('end', () => {
+    clearInterval(interval);
+    printProgress();
+    process.stderr.write('\nFinished downloading assets\n');
+  });
+}
+
+function getApiKeyOrThrow(): string {
+  const auth = readAuth();
+  if (!auth?.apiKey) {
+    throw new Error('Not logged in. Please provide apiKey or run: sharetribe-cli login');
+  }
+  return auth.apiKey;
+}
+
+function getAssetsPullUrl(marketplace: string, version?: string): URL {
+  const url = new URL(getApiBaseUrl() + '/assets/pull');
+  url.searchParams.set('marketplace', marketplace);
+  if (version) {
+    url.searchParams.set('version', version);
+  } else {
+    url.searchParams.set('version-alias', 'latest');
+  }
+  return url;
+}
+
+function getErrorMessage(body: string, statusCode: number): string {
+  try {
+    const parsed = JSON.parse(body) as { errors?: Array<{ message?: string }> };
+    const message = parsed.errors?.[0]?.message;
+    if (message) {
+      return message;
+    }
+  } catch {
+    // Ignore JSON parse errors
+  }
+  return body || `HTTP ${statusCode}`;
+}
+
+async function getAssetsZipStream(
+  marketplace: string,
+  version?: string
+): Promise<http.IncomingMessage> {
+  const url = getAssetsPullUrl(marketplace, version);
+  const apiKey = getApiKeyOrThrow();
+  const isHttps = url.protocol === 'https:';
+  const client = isHttps ? https : http;
+
+  return new Promise((resolve, reject) => {
+    const req = client.request(
+      {
+        method: 'GET',
+        hostname: url.hostname,
+        port: url.port || (isHttps ? 443 : 80),
+        path: url.pathname + url.search,
+        headers: {
+          Authorization: `Apikey ${apiKey}`,
+          Accept: 'application/zip',
+        },
+      },
+      (res) => {
+        const statusCode = res.statusCode || 0;
+        if (statusCode < 200 || statusCode >= 300) {
+          const chunks: Buffer[] = [];
+          res.on('data', (chunk: Buffer) => chunks.push(chunk));
+          res.on('end', () => {
+            const body = Buffer.concat(chunks).toString('utf-8');
+            reject(new Error(getErrorMessage(body, statusCode)));
+          });
+          return;
+        }
+        resolve(res);
+      }
+    );
+
+    req.setTimeout(120000, () => {
+      req.destroy(new Error('Request timeout'));
+    });
+    req.on('error', reject);
+    req.end();
+  });
+}
+
+function createTempZipPath(): string {
+  return join(tmpdir(), `assets-${Date.now()}.zip`);
+}
+
+function removeAssetsDir(filename: string): string {
+  if (filename.startsWith(ASSETS_DIR)) {
+    return filename.slice(ASSETS_DIR.length);
+  }
+  return filename;
+}
+
+function readStreamToString(stream: NodeJS.ReadableStream): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    stream.on('data', (chunk: Buffer) => chunks.push(chunk));
+    stream.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
+    stream.on('error', reject);
+  });
+}
+
+async function unzipAssets(zipPath: string, basePath: string): Promise<AssetMetadata> {
+  return new Promise((resolve, reject) => {
+    yauzl.open(zipPath, { lazyEntries: true }, (err, zipfile) => {
+      if (err || !zipfile) {
+        reject(err || new Error('Failed to open zip file'));
+        return;
+      }
+
+      let assetMeta: AssetMetadata | null = null;
+
+      zipfile.on('error', reject);
+      zipfile.on('end', () => {
+        if (!assetMeta) {
+          reject(new Error('Asset metadata not found in zip'));
+          return;
+        }
+        resolve(assetMeta);
+      });
+
+      zipfile.readEntry();
+      zipfile.on('entry', (entry) => {
+        if (entry.fileName.endsWith('/')) {
+          zipfile.readEntry();
+          return;
+        }
+
+        zipfile.openReadStream(entry, (streamErr, readStream) => {
+          if (streamErr || !readStream) {
+            reject(streamErr || new Error('Failed to read zip entry'));
+            return;
+          }
+
+          if (entry.fileName === ASSET_META_FILENAME) {
+            readStreamToString(readStream)
+              .then((content) => {
+                assetMeta = parseAssetMetadataEdn(content);
+                if (!assetMeta) {
+                  reject(new Error('Invalid asset metadata'));
+                  return;
+                }
+                zipfile.readEntry();
+              })
+              .catch(reject);
+            return;
+          }
+
+          const assetPath = join(basePath, removeAssetsDir(entry.fileName));
+          const assetDir = dirname(assetPath);
+          if (!existsSync(assetDir)) {
+            mkdirSync(assetDir, { recursive: true });
+          }
+
+          pipeline(readStream, createWriteStream(assetPath))
+            .then(() => zipfile.readEntry())
+            .catch(reject);
+        });
+      });
+    });
+  });
+}
+
 /**
  * Pulls assets from remote
  */
@@ -141,7 +383,7 @@ async function pullAssets(
   prune?: boolean
 ): Promise<void> {
   try {
-    // Validate path
+    // Create directory if it doesn't exist
     if (!existsSync(path)) {
       mkdirSync(path, { recursive: true });
     }
@@ -151,57 +393,48 @@ async function pullAssets(
       throw new Error(`${path} is not a directory`);
     }
 
-    // Fetch assets from API
-    const result = await sdkPullAssets(undefined, marketplace, version ? { version } : undefined);
-    const remoteVersion = result.version;
-
-    // Read current metadata
+    const localAssets = prune ? listLocalAssetPaths(path) : [];
     const currentMeta = readAssetMetadata(path);
+    const tempZipPath = createTempZipPath();
 
-    // Check if up to date
-    if (currentMeta && currentMeta.version === remoteVersion && result.assets.length === currentMeta.assets.length) {
-      console.log('Assets are up to date.');
-      return;
-    }
+    try {
+      const zipStream = await getAssetsZipStream(marketplace, version);
+      printDownloadProgress(zipStream);
+      await pipeline(zipStream, createWriteStream(tempZipPath));
 
-    // Write assets to disk
-    const newAssets: Array<{ path: string; 'content-hash': string }> = [];
-    for (const asset of result.assets) {
-      const assetPath = join(path, asset.path);
-      const assetDir = dirname(assetPath);
+      const newAssetMeta = await unzipAssets(tempZipPath, path);
+      const remoteVersion = newAssetMeta.version;
 
-      if (!existsSync(assetDir)) {
-        mkdirSync(assetDir, { recursive: true });
-      }
+      const deletedPaths = prune
+        ? new Set(localAssets.filter(p => !newAssetMeta.assets.some(a => a.path === p)))
+        : new Set<string>();
 
-      // Decode base64 data
-      const data = Buffer.from(asset.dataRaw, 'base64');
-      writeFileSync(assetPath, data);
+      const updated = currentMeta?.version !== remoteVersion;
+      const shouldReportUpdate = updated || deletedPaths.size > 0;
 
-      const hash = calculateHash(data);
-      newAssets.push({ path: asset.path, 'content-hash': asset.contentHash || hash });
-    }
-
-    // Prune deleted assets if requested
-    if (prune && currentMeta) {
-      const remotePaths = new Set(result.assets.map(a => a.path));
-      for (const localAsset of currentMeta.assets) {
-        if (!remotePaths.has(localAsset.path)) {
-          const assetPath = join(path, localAsset.path);
-          if (existsSync(assetPath)) {
-            unlinkSync(assetPath);
+      if (deletedPaths.size > 0) {
+        for (const assetPath of deletedPaths) {
+          const fullPath = join(path, assetPath);
+          if (existsSync(fullPath)) {
+            unlinkSync(fullPath);
           }
         }
       }
-    }
 
-    // Update metadata
-    writeAssetMetadata(path, {
-      version: remoteVersion,
-      assets: newAssets,
-    });
-
-    console.log(`Version ${remoteVersion} successfully pulled.`);
+      if (shouldReportUpdate) {
+        writeAssetMetadata(path, {
+          version: remoteVersion,
+          assets: newAssetMeta.assets,
+        });
+        console.log(`Version ${remoteVersion} successfully pulled.`);
+      } else {
+        console.log('Assets are up to date.');
+      }
+    } finally {
+      if (existsSync(tempZipPath)) {
+        unlinkSync(tempZipPath);
+      }
+    }
   } catch (error) {
     if (error && typeof error === 'object' && 'message' in error) {
       printError(error.message as string);
@@ -212,6 +445,22 @@ async function pullAssets(
   }
 }
 
+/**
+ * Filters assets to only those that have changed
+ */
+function filterChangedAssets(
+  existingMeta: Array<{ path: string; 'content-hash': string }>,
+  localAssets: Array<{ path: string; hash: string }>
+): Array<{ path: string; data: Buffer; hash: string }> {
+  const hashByPath = new Map(existingMeta.map(a => [a.path, a['content-hash']]));
+  
+  return localAssets.filter(asset => {
+    const storedHash = hashByPath.get(asset.path);
+    // Assets without stored metadata are treated as changed
+    return !storedHash || storedHash !== asset.hash;
+  });
+}
+
 /**
  * Pushes assets to remote
  */
@@ -236,33 +485,23 @@ async function pushAssets(
     // Validate JSON files
     validateJsonAssets(localAssets);
 
-    // Build operations
-    const operations: Array<{
-      path: string;
-      op: 'upsert' | 'delete';
-      data?: Buffer;
-    }> = [];
+    // Filter to only changed assets
+    const changedAssets = filterChangedAssets(currentMeta?.assets || [], localAssets);
 
-    // Find assets to upsert (new or changed)
-    const localAssetMap = new Map(localAssets.map(a => [a.path, a]));
-    const currentAssetMap = new Map((currentMeta?.assets || []).map(a => [a.path, a['content-hash']]));
-
-    for (const [assetPath, asset] of localAssetMap) {
-      const currentHash = currentAssetMap.get(assetPath);
-      if (!currentHash || currentHash !== asset.hash) {
-        operations.push({
-          path: assetPath,
-          op: 'upsert',
-          data: asset.data,
-        });
-      }
-    }
+    // Separate JSON and non-JSON assets
+    const isJsonAsset = (assetPath: string): boolean => {
+      return assetPath.toLowerCase().endsWith('.json');
+    };
+
+    const stageableAssets = changedAssets.filter(a => !isJsonAsset(a.path));
 
     // Find assets to delete (if prune enabled)
+    const localAssetMap = new Map(localAssets.map(a => [a.path, a]));
+    const deleteOperations: Array<{ path: string; op: 'delete' }> = [];
     if (prune && currentMeta) {
       for (const currentAsset of currentMeta.assets) {
         if (!localAssetMap.has(currentAsset.path)) {
-          operations.push({
+          deleteOperations.push({
             path: currentAsset.path,
             op: 'delete',
           });
@@ -271,20 +510,62 @@ async function pushAssets(
     }
 
     // Check if there are any changes
-    if (operations.length === 0) {
+    const noOps = changedAssets.length === 0 && deleteOperations.length === 0;
+    if (noOps) {
       console.log('Assets are up to date.');
       return;
     }
 
-    const changedAssetPaths = operations
-      .filter(op => op.op === 'upsert')
-      .map(op => op.path);
-    if (changedAssetPaths.length > 0) {
-      console.log(chalk.green(`Uploading changed assets: ${changedAssetPaths.join(', ')}`));
+    // Log changed assets
+    if (changedAssets.length > 0) {
+      const paths = changedAssets.map(a => a.path).join(', ');
+      console.log(chalk.green(`Uploading changed assets: ${paths}`));
     }
 
+    // Stage non-JSON assets
+    const stagedByPath = new Map<string, string>();
+    if (stageableAssets.length > 0) {
+      const paths = stageableAssets.map(a => a.path).join(', ');
+      console.log(chalk.green(`Staging assets: ${paths}`));
+
+      for (const asset of stageableAssets) {
+        try {
+          const stagingResult = await sdkStageAsset(
+            undefined,
+            marketplace,
+            asset.data,
+            asset.path
+          );
+          stagedByPath.set(asset.path, stagingResult.stagingId);
+        } catch (error) {
+          if (error && typeof error === 'object' && 'code' in error && error.code === 'asset-invalid-content') {
+            const detail = 'message' in error ? error.message : 'The file is missing or uses an unsupported format.';
+            throw new Error(`Failed to stage image ${asset.path}: ${detail}\nFix the file and rerun assets push to retry staging.`);
+          }
+          throw error;
+        }
+      }
+    }
+
+    // Build upsert operations
+    const upsertOperations = changedAssets.map(asset => {
+      const stagingId = stagedByPath.get(asset.path);
+      return {
+        path: asset.path,
+        op: 'upsert' as const,
+        ...(stagingId
+          ? { stagingId }
+          : { data: asset.data, filename: asset.path }),
+      };
+    });
+
     // Upload to API
-    const result = await sdkPushAssets(undefined, marketplace, currentVersion, operations);
+    const result = await sdkPushAssets(
+      undefined,
+      marketplace,
+      currentVersion,
+      [...upsertOperations, ...deleteOperations]
+    );
 
     // Update local metadata
     writeAssetMetadata(path, {
@@ -345,3 +626,9 @@ export function registerAssetsCommands(program: Command): void {
       await pushAssets(marketplace, opts.path, opts.prune);
     });
 }
+
+export const __test__ = {
+  formatDownloadProgress,
+  removeAssetsDir,
+  parseAssetMetadataEdn,
+};

package/src/commands/debug.ts

@@ -2,7 +2,6 @@
  * Debug command - display config and auth info
  */
 
-import edn from 'jsedn';
 import { getConfigMap, readAuth } from 'sharetribe-flex-build-sdk';
 
 function maskLast4(value: string): string {
@@ -12,25 +11,16 @@ function maskLast4(value: string): string {
   return `...${value.slice(-4)}`;
 }
 
-function toEdnMap(record: Record<string, string>): edn.Map {
-  const entries: Array<unknown> = [];
-  for (const [key, value] of Object.entries(record)) {
-    entries.push(edn.kw(`:${key}`), value);
-  }
-  return new edn.Map(entries);
-}
-
 export function debug(): void {
   const auth = readAuth();
   const apiKey = auth?.apiKey ? maskLast4(auth.apiKey) : 'No API key set';
   const confMap = getConfigMap();
 
-  const payload = new edn.Map([
-    edn.kw(':api-key'),
-    apiKey,
-    edn.kw(':conf-map'),
-    toEdnMap(confMap),
-  ]);
+  const confMapEntries = Object.keys(confMap)
+    .sort()
+    .map((key) => `:${key} ${confMap[key]}`)
+    .join(' ');
+  const confMapFormatted = confMapEntries ? `{${confMapEntries}}` : '{}';
 
-  console.log(edn.encode(payload));
+  console.log(`{:api-key ${apiKey}, :conf-map ${confMapFormatted}}`);
 }

package/src/commands/search/index.ts

@@ -208,12 +208,18 @@ export function registerSearchCommands(program: Command): void {
   searchCmd
     .command('set')
     .description('set search schema')
-    .requiredOption('--key <KEY>', 'schema key')
-    .requiredOption('--scope <SCOPE>', 'schema scope')
-    .requiredOption('--type <TYPE>', 'value type (enum, multi-enum, boolean, long, or text)')
+    .requiredOption('--key <KEY>', 'key name')
+    .requiredOption(
+      '--scope <SCOPE>',
+      'extended data scope (either metadata or public for listing schema, metadata, private, protected or public for userProfile schema, metadata or protected for transaction schema)'
+    )
+    .requiredOption('--type <TYPE>', 'value type (either enum, multi-enum, boolean, long or text)')
     .option('--doc <DOC>', 'description of the schema')
     .option('--default <DEFAULT>', 'default value for search if value is not set')
-    .option('--schema-for <SCHEMA_FOR>', 'subject of the schema (listing, userProfile, or transaction)')
+    .option(
+      '--schema-for <SCHEMA_FOR>',
+      'Subject of the schema (either listing, userProfile or transaction, defaults to listing)'
+    )
     .option('-m, --marketplace <MARKETPLACE_ID>', 'marketplace identifier')
     .action(async (opts) => {
       const marketplace = opts.marketplace || program.opts().marketplace;
@@ -235,9 +241,15 @@ export function registerSearchCommands(program: Command): void {
   searchCmd
     .command('unset')
     .description('unset search schema')
-    .requiredOption('--key <KEY>', 'schema key')
-    .requiredOption('--scope <SCOPE>', 'schema scope')
-    .option('--schema-for <SCHEMA_FOR>', 'subject of the schema (listing, userProfile, or transaction)')
+    .requiredOption('--key <KEY>', 'key name')
+    .requiredOption(
+      '--scope <SCOPE>',
+      'extended data scope (either metadata or public for listing schema, metadata, private, protected or public for userProfile schema, metadata or protected for transaction schema)'
+    )
+    .option(
+      '--schema-for <SCHEMA_FOR>',
+      'Subject of the schema (either listing, userProfile or transaction, defaults to listing)'
+    )
     .option('-m, --marketplace <MARKETPLACE_ID>', 'marketplace identifier')
     .action(async (opts) => {
       const marketplace = opts.marketplace || program.opts().marketplace;