storyblok 4.17.8 → 4.17.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -221,7 +221,9 @@ type DeepPartial<T> = T extends object ? {
221
221
 
222
222
  interface ApiConfig {
223
223
  maxRetries: number;
224
- maxConcurrency: number;
224
+ rateLimit: number;
225
+ /** @deprecated Use `rateLimit` instead. @todo(next-major): Remove this field. */
226
+ maxConcurrency?: number;
225
227
  }
226
228
  interface LogConsoleConfig {
227
229
  enabled: boolean;
@@ -221,7 +221,9 @@ type DeepPartial<T> = T extends object ? {
221
221
 
222
222
  interface ApiConfig {
223
223
  maxRetries: number;
224
- maxConcurrency: number;
224
+ rateLimit: number;
225
+ /** @deprecated Use `rateLimit` instead. @todo(next-major): Remove this field. */
226
+ maxConcurrency?: number;
225
227
  }
226
228
  interface LogConsoleConfig {
227
229
  enabled: boolean;
@@ -1 +1 @@
1
- {"version":3,"file":"index.mjs","sources":["../../src/lib/config/types.ts"],"sourcesContent":["import type { RegionCode } from '../../constants';\nimport type { Command, Option } from 'commander';\nimport type { CommandOptions } from '../../types';\nimport type { PullComponentsOptions } from '../../commands/components/pull/constants';\nimport type { PushComponentsOptions } from '../../commands/components/push/constants';\nimport type { PullDatasourcesOptions } from '../../commands/datasources/pull/constants';\nimport type { PushDatasourcesOptions } from '../../commands/datasources/push/constants';\nimport type { DeleteDatasourceOptions } from '../../commands/datasources/delete/constants';\nimport type { MigrationsGenerateOptions } from '../../commands/migrations/generate/constants';\nimport type { MigrationsRunOptions } from '../../commands/migrations/run/constants';\nimport type { MigrationsRollbackOptions } from '../../commands/migrations/rollback/types';\nimport type { GenerateTypesOptions } from '../../commands/types/generate/constants';\nimport type { PullLanguagesOptions } from '../../commands/languages/constants';\nimport type { PullAssetsOptions } from '../../commands/assets/pull/types';\nimport type { PushAssetsOptions } from '../../commands/assets/push/types';\nimport type { PullStoriesOptions } from '../../commands/stories/pull/types';\nimport type { PushStoriesOptions } from '../../commands/stories/push/types';\nimport type { PruneLogsOptions } from '../../commands/logs/prune/types';\nimport type { PruneReportsOptions } from '../../commands/reports/prune/types';\nimport type { DeepPartial } from '../../utils/types';\n\nexport type PlainObject = Record<string, any>;\n\nexport interface ApiConfig {\n maxRetries: number;\n maxConcurrency: number;\n}\n\nexport interface LogConsoleConfig {\n enabled: boolean;\n level: string;\n}\n\nexport interface LogFileConfig {\n enabled: boolean;\n level: string;\n maxFiles: number;\n}\n\nexport interface LogConfig {\n console: LogConsoleConfig;\n file: LogFileConfig;\n}\n\nexport interface ReportConfig {\n enabled: boolean;\n maxFiles: number;\n}\n\nexport interface UiConfig {\n enabled: boolean;\n}\n\nexport interface GlobalConfig {\n region?: RegionCode;\n space?: number | string;\n path?: string;\n api: ApiConfig;\n log: LogConfig;\n report: ReportConfig;\n ui: UiConfig;\n verbose: boolean;\n}\n\nexport interface ResolvedCliConfig extends GlobalConfig {\n [key: string]: any;\n}\n\ntype StripCommandOption<T> = T extends CommandOptions ? Omit<T, keyof CommandOptions> : T;\ntype CommandConfig<T> = Partial<StripCommandOption<T>>;\n\nexport interface SharedConfigOptions {\n space?: number | string;\n path?: string;\n}\n\nexport interface CommandOptionsMap {\n components: { pull: PullComponentsOptions; push: PushComponentsOptions };\n datasources: { pull: PullDatasourcesOptions; push: PushDatasourcesOptions; delete: DeleteDatasourceOptions };\n migrations: { generate: MigrationsGenerateOptions; run: MigrationsRunOptions; rollback: MigrationsRollbackOptions };\n types: { generate: GenerateTypesOptions };\n languages: { pull: PullLanguagesOptions };\n assets: { pull: PullAssetsOptions; push: PushAssetsOptions };\n stories: { pull: PullStoriesOptions; push: PushStoriesOptions };\n logs: { list: Record<string, never>; prune: PruneLogsOptions };\n reports: { list: Record<string, never>; prune: PruneReportsOptions };\n}\n\ntype SubcommandConfig<T> = SharedConfigOptions & CommandConfig<T>;\n\ntype ModuleConfig<Subs extends Record<string, any>> = SharedConfigOptions & {\n [K in keyof Subs]?: SubcommandConfig<Subs[K]>;\n};\n\nexport type ModulesConfig = {\n [K in keyof CommandOptionsMap]?: ModuleConfig<CommandOptionsMap[K]>;\n};\n\nexport type ComponentsModuleConfig = ModulesConfig['components'];\nexport type DatasourcesModuleConfig = ModulesConfig['datasources'];\nexport type MigrationsModuleConfig = ModulesConfig['migrations'];\nexport type TypesModuleConfig = ModulesConfig['types'];\nexport type LanguagesModuleConfig = ModulesConfig['languages'];\nexport type AssetsModuleConfig = ModulesConfig['assets'];\nexport type StoriesModuleConfig = ModulesConfig['stories'];\nexport type LogsModuleConfig = ModulesConfig['logs'];\nexport type ReportsModuleConfig = ModulesConfig['reports'];\n\nexport interface StoryblokConfig extends DeepPartial<GlobalConfig> {\n modules?: ModulesConfig;\n}\n\nexport function defineConfig<T extends StoryblokConfig>(config: T): T {\n return config;\n}\n\nexport type OptionParser<T> = (value: string, previous?: T) => T;\n\nexport interface GlobalOptionDefinition<T = unknown> {\n flags: string;\n description: string;\n defaultValue?: T;\n parser?: OptionParser<T>;\n}\n\nexport interface ConfigLocation {\n cwd: string;\n configFile: string;\n}\n\nexport type CommanderOption = Option;\nexport type CommanderCommand = Command;\n"],"names":[],"mappings":"AAgHO,SAAS,aAAwC,MAAA,EAAc;AACpE,EAAA,OAAO,MAAA;AACT;;;;"}
1
+ {"version":3,"file":"index.mjs","sources":["../../src/lib/config/types.ts"],"sourcesContent":["import type { RegionCode } from '../../constants';\nimport type { Command, Option } from 'commander';\nimport type { CommandOptions } from '../../types';\nimport type { PullComponentsOptions } from '../../commands/components/pull/constants';\nimport type { PushComponentsOptions } from '../../commands/components/push/constants';\nimport type { PullDatasourcesOptions } from '../../commands/datasources/pull/constants';\nimport type { PushDatasourcesOptions } from '../../commands/datasources/push/constants';\nimport type { DeleteDatasourceOptions } from '../../commands/datasources/delete/constants';\nimport type { MigrationsGenerateOptions } from '../../commands/migrations/generate/constants';\nimport type { MigrationsRunOptions } from '../../commands/migrations/run/constants';\nimport type { MigrationsRollbackOptions } from '../../commands/migrations/rollback/types';\nimport type { GenerateTypesOptions } from '../../commands/types/generate/constants';\nimport type { PullLanguagesOptions } from '../../commands/languages/constants';\nimport type { PullAssetsOptions } from '../../commands/assets/pull/types';\nimport type { PushAssetsOptions } from '../../commands/assets/push/types';\nimport type { PullStoriesOptions } from '../../commands/stories/pull/types';\nimport type { PushStoriesOptions } from '../../commands/stories/push/types';\nimport type { PruneLogsOptions } from '../../commands/logs/prune/types';\nimport type { PruneReportsOptions } from '../../commands/reports/prune/types';\nimport type { DeepPartial } from '../../utils/types';\n\nexport type PlainObject = Record<string, any>;\n\nexport interface ApiConfig {\n maxRetries: number;\n rateLimit: number;\n /** @deprecated Use `rateLimit` instead. @todo(next-major): Remove this field. */\n maxConcurrency?: number;\n}\n\nexport interface LogConsoleConfig {\n enabled: boolean;\n level: string;\n}\n\nexport interface LogFileConfig {\n enabled: boolean;\n level: string;\n maxFiles: number;\n}\n\nexport interface LogConfig {\n console: LogConsoleConfig;\n file: LogFileConfig;\n}\n\nexport interface ReportConfig {\n enabled: boolean;\n maxFiles: number;\n}\n\nexport interface UiConfig {\n enabled: boolean;\n}\n\nexport interface GlobalConfig {\n region?: RegionCode;\n space?: number | string;\n path?: string;\n api: ApiConfig;\n log: LogConfig;\n report: ReportConfig;\n ui: UiConfig;\n verbose: boolean;\n}\n\nexport interface ResolvedCliConfig extends GlobalConfig {\n [key: string]: any;\n}\n\ntype StripCommandOption<T> = T extends CommandOptions ? Omit<T, keyof CommandOptions> : T;\ntype CommandConfig<T> = Partial<StripCommandOption<T>>;\n\nexport interface SharedConfigOptions {\n space?: number | string;\n path?: string;\n}\n\nexport interface CommandOptionsMap {\n components: { pull: PullComponentsOptions; push: PushComponentsOptions };\n datasources: { pull: PullDatasourcesOptions; push: PushDatasourcesOptions; delete: DeleteDatasourceOptions };\n migrations: { generate: MigrationsGenerateOptions; run: MigrationsRunOptions; rollback: MigrationsRollbackOptions };\n types: { generate: GenerateTypesOptions };\n languages: { pull: PullLanguagesOptions };\n assets: { pull: PullAssetsOptions; push: PushAssetsOptions };\n stories: { pull: PullStoriesOptions; push: PushStoriesOptions };\n logs: { list: Record<string, never>; prune: PruneLogsOptions };\n reports: { list: Record<string, never>; prune: PruneReportsOptions };\n}\n\ntype SubcommandConfig<T> = SharedConfigOptions & CommandConfig<T>;\n\ntype ModuleConfig<Subs extends Record<string, any>> = SharedConfigOptions & {\n [K in keyof Subs]?: SubcommandConfig<Subs[K]>;\n};\n\nexport type ModulesConfig = {\n [K in keyof CommandOptionsMap]?: ModuleConfig<CommandOptionsMap[K]>;\n};\n\nexport type ComponentsModuleConfig = ModulesConfig['components'];\nexport type DatasourcesModuleConfig = ModulesConfig['datasources'];\nexport type MigrationsModuleConfig = ModulesConfig['migrations'];\nexport type TypesModuleConfig = ModulesConfig['types'];\nexport type LanguagesModuleConfig = ModulesConfig['languages'];\nexport type AssetsModuleConfig = ModulesConfig['assets'];\nexport type StoriesModuleConfig = ModulesConfig['stories'];\nexport type LogsModuleConfig = ModulesConfig['logs'];\nexport type ReportsModuleConfig = ModulesConfig['reports'];\n\nexport interface StoryblokConfig extends DeepPartial<GlobalConfig> {\n modules?: ModulesConfig;\n}\n\nexport function defineConfig<T extends StoryblokConfig>(config: T): T {\n return config;\n}\n\nexport type OptionParser<T> = (value: string, previous?: T) => T;\n\nexport interface GlobalOptionDefinition<T = unknown> {\n flags: string;\n description: string;\n defaultValue?: T;\n parser?: OptionParser<T>;\n}\n\nexport interface ConfigLocation {\n cwd: string;\n configFile: string;\n}\n\nexport type CommanderOption = Option;\nexport type CommanderCommand = Command;\n"],"names":[],"mappings":"AAkHO,SAAS,aAAwC,MAAA,EAAc;AACpE,EAAA,OAAO,MAAA;AACT;;;;"}
package/dist/index.mjs CHANGED
@@ -186,7 +186,7 @@ const BASE_GLOBAL_CONFIG = {
186
186
  path: void 0,
187
187
  api: {
188
188
  maxRetries: 3,
189
- maxConcurrency: 6
189
+ rateLimit: 6
190
190
  },
191
191
  log: {
192
192
  console: {
@@ -445,10 +445,16 @@ const GLOBAL_OPTION_DEFINITIONS = [
445
445
  defaultValue: DEFAULT_GLOBAL_CONFIG.api.maxRetries,
446
446
  parser: parseNumber
447
447
  },
448
+ {
449
+ flags: "--api-rate-limit <number>",
450
+ description: "Maximum MAPI requests per second (default: 6)",
451
+ defaultValue: DEFAULT_GLOBAL_CONFIG.api.rateLimit,
452
+ parser: parseNumber
453
+ },
454
+ // @todo(next-major): Remove deprecated --api-max-concurrency flag.
448
455
  {
449
456
  flags: "--api-max-concurrency <number>",
450
- description: "Maximum concurrent API requests executed by the CLI",
451
- defaultValue: DEFAULT_GLOBAL_CONFIG.api.maxConcurrency,
457
+ description: "[Deprecated] Use --api-rate-limit instead.",
452
458
  parser: parseNumber
453
459
  },
454
460
  // Boolean flags that default to true need both positive and negative forms
@@ -564,6 +570,11 @@ async function resolveConfig(thisCommand, ancestry) {
564
570
  const knownModuleKeys = getModuleNames(root);
565
571
  for (const layer of layers) {
566
572
  const { modules, ...globalLayer } = layer;
573
+ if (globalLayer.api?.maxConcurrency !== void 0) {
574
+ console.warn("[storyblok] Config option `api.maxConcurrency` is deprecated. Use `api.rateLimit` instead.");
575
+ globalLayer.api.rateLimit = globalLayer.api.maxConcurrency;
576
+ delete globalLayer.api.maxConcurrency;
577
+ }
567
578
  mergeDeep(globalResolved, globalLayer);
568
579
  if (modules && isPlainObject(modules)) {
569
580
  warnUnknownModuleKeys(modules, knownModuleKeys);
@@ -571,6 +582,12 @@ async function resolveConfig(thisCommand, ancestry) {
571
582
  }
572
583
  }
573
584
  applyCliOverrides(commandChain, globalResolved, localResolved);
585
+ const globalResolvedApi = globalResolved.api;
586
+ if (globalResolvedApi?.maxConcurrency !== void 0) {
587
+ console.warn("[storyblok] CLI flag `--api-max-concurrency` is deprecated. Use `--api-rate-limit` instead.");
588
+ globalResolvedApi.rateLimit = globalResolvedApi.maxConcurrency;
589
+ delete globalResolvedApi.maxConcurrency;
590
+ }
574
591
  const resolved = structuredClone(defaultConfig);
575
592
  mergeDeep(resolved, globalResolved);
576
593
  Object.assign(resolved, localResolved);
@@ -696,6 +713,8 @@ const API_ACTIONS = {
696
713
  push_asset_folder: "Failed to push asset folder",
697
714
  push_asset_create: "Failed to create asset",
698
715
  push_asset_update: "Failed to update asset",
716
+ pull_asset_internal_tags: "Failed to pull asset internal tags",
717
+ push_asset_internal_tag: "Failed to push asset internal tag",
699
718
  pull_datasources: "Failed to pull datasources",
700
719
  push_datasource: "Failed to push datasource",
701
720
  update_datasource: "Failed to update datasource",
@@ -1832,7 +1851,7 @@ function createMapiClient(options) {
1832
1851
  const { api } = getActiveConfig();
1833
1852
  return createManagementApiClient({
1834
1853
  ...options,
1835
- rateLimit: options.rateLimit ?? (api.maxConcurrency > 0 ? { maxConcurrency: api.maxConcurrency } : false)
1854
+ rateLimit: options.rateLimit ?? (api.rateLimit > 0 ? api.rateLimit : false)
1836
1855
  });
1837
1856
  }
1838
1857
  function getMapiClient(options) {
@@ -3573,17 +3592,17 @@ class ProgressDisplay {
3573
3592
  }
3574
3593
  const progressDisplay = new ProgressDisplay();
3575
3594
 
3576
- async function processAllResources(graph, space, maxConcurrency = getActiveConfig().api.maxConcurrency) {
3595
+ async function processAllResources(graph, space, backpressure = getActiveConfig().api.rateLimit) {
3577
3596
  const levels = determineProcessingOrder(graph);
3578
3597
  const results = { successful: [], failed: [] };
3579
3598
  const totalResources = levels.reduce((sum, level) => sum + level.nodes.length, 0);
3580
3599
  progressDisplay.start(totalResources);
3581
3600
  for (const level of levels) {
3582
3601
  if (level.isCyclic) {
3583
- const cyclicResults = await processCyclicLevel(level, graph, space, maxConcurrency);
3602
+ const cyclicResults = await processCyclicLevel(level, graph, space, backpressure);
3584
3603
  mergeResults(results, cyclicResults);
3585
3604
  } else {
3586
- const levelResults = await processLevel(level.nodes, graph, space, maxConcurrency);
3605
+ const levelResults = await processLevel(level.nodes, graph, space, backpressure);
3587
3606
  mergeResults(results, levelResults);
3588
3607
  }
3589
3608
  }
@@ -3598,12 +3617,12 @@ async function processAllResources(graph, space, maxConcurrency = getActiveConfi
3598
3617
  });
3599
3618
  return results;
3600
3619
  }
3601
- async function processCyclicLevel(level, graph, space, maxConcurrency) {
3620
+ async function processCyclicLevel(level, graph, space, backpressure) {
3602
3621
  progressDisplay.clearProgress();
3603
3622
  console.log(`
3604
3623
  \u{1F504} Detected circular dependencies: ${level.nodes.map((id) => id.replace("component:", "")).join(", ")}`);
3605
3624
  await createStubComponents(level.nodes, graph, space);
3606
- return await processLevel(level.nodes, graph, space, maxConcurrency);
3625
+ return await processLevel(level.nodes, graph, space, backpressure);
3607
3626
  }
3608
3627
  async function createStubComponents(nodeIds, graph, space) {
3609
3628
  const missingComponents = [];
@@ -3643,17 +3662,17 @@ function createMinimalStubComponent(name) {
3643
3662
  // Minimal empty schema
3644
3663
  };
3645
3664
  }
3646
- async function processLevel(level, graph, space, maxConcurrency) {
3665
+ async function processLevel(level, graph, space, backpressure) {
3647
3666
  for (const nodeId of level) {
3648
3667
  const node = graph.nodes.get(nodeId);
3649
3668
  node.resolveReferences(graph);
3650
3669
  }
3651
- const semaphore = Array.from({ length: maxConcurrency }, () => null);
3670
+ const semaphore = Array.from({ length: backpressure }, () => null);
3652
3671
  const promises = [];
3653
3672
  for (let i = 0; i < level.length; i++) {
3654
3673
  const nodeId = level[i];
3655
- const slotIndex = i % maxConcurrency;
3656
- if (i >= maxConcurrency && semaphore[slotIndex]) {
3674
+ const slotIndex = i % backpressure;
3675
+ if (i >= backpressure && semaphore[slotIndex]) {
3657
3676
  await semaphore[slotIndex];
3658
3677
  }
3659
3678
  const promise = processNode(nodeId, graph, space);
@@ -3734,11 +3753,11 @@ function getResourceTypeColor(type) {
3734
3753
  }
3735
3754
  }
3736
3755
 
3737
- async function pushWithDependencyGraph(space, spaceState, maxConcurrency = getActiveConfig().api.maxConcurrency) {
3756
+ async function pushWithDependencyGraph(space, spaceState, backpressure = getActiveConfig().api.rateLimit) {
3738
3757
  const context = { spaceState };
3739
3758
  const graph = buildDependencyGraph(context);
3740
3759
  validateGraph(graph);
3741
- const results = await processAllResources(graph, space, maxConcurrency);
3760
+ const results = await processAllResources(graph, space, backpressure);
3742
3761
  return results;
3743
3762
  }
3744
3763
 
@@ -4255,6 +4274,14 @@ const prefetchTargetStoriesByKeys = async (spaceId, keys, options) => {
4255
4274
  return result;
4256
4275
  };
4257
4276
 
4277
+ const PIPELINE_BACKPRESSURE_MULTIPLIER = 2;
4278
+ const DEFAULT_PIPELINE_BACKPRESSURE = 12;
4279
+ function createPipelineBackpressureLock(limit) {
4280
+ const rateLimit = getActiveConfig().api.rateLimit;
4281
+ const n = (rateLimit > 0 ? rateLimit * PIPELINE_BACKPRESSURE_MULTIPLIER : DEFAULT_PIPELINE_BACKPRESSURE);
4282
+ return new Sema(n);
4283
+ }
4284
+
4258
4285
  const ERROR_CODES = {
4259
4286
  MIGRATION_APPLY_TO_STORY_ERROR: "MIGRATION_APPLY_TO_STORY_ERROR",
4260
4287
  MIGRATION_CREATE_STORIES_PIPELINE_ERROR: "MIGRATION_CREATE_STORIES_PIPELINE_ERROR",
@@ -4319,14 +4346,13 @@ async function* storiesIterator(spaceId, params, onTotal) {
4319
4346
  }
4320
4347
  }
4321
4348
  class StoriesStream extends Transform {
4322
- constructor(spaceId, batchSize, onProgress) {
4349
+ constructor(spaceId, onProgress) {
4323
4350
  super({
4324
4351
  objectMode: true
4325
4352
  });
4326
4353
  this.spaceId = spaceId;
4327
- this.batchSize = batchSize;
4328
4354
  this.onProgress = onProgress;
4329
- this.semaphore = new Sema(this.batchSize);
4355
+ this.semaphore = createPipelineBackpressureLock();
4330
4356
  }
4331
4357
  semaphore;
4332
4358
  async _transform(chunk, _encoding, callback) {
@@ -4354,13 +4380,12 @@ class StoriesStream extends Transform {
4354
4380
  const createStoriesStream = async ({
4355
4381
  spaceId,
4356
4382
  params,
4357
- batchSize = 100,
4358
4383
  onTotal,
4359
4384
  onProgress
4360
4385
  }) => {
4361
4386
  const iterator = storiesIterator(spaceId, params, onTotal);
4362
4387
  const listStoriesStream = Readable.from(iterator);
4363
- return pipeline(listStoriesStream, new StoriesStream(spaceId, batchSize, onProgress), (err) => {
4388
+ return pipeline(listStoriesStream, new StoriesStream(spaceId, onProgress), (err) => {
4364
4389
  if (err) {
4365
4390
  console.error(err);
4366
4391
  getLogger().error(err.message, { errorCode: ERROR_CODES.MIGRATION_CREATE_STORIES_PIPELINE_ERROR });
@@ -4739,16 +4764,14 @@ class UpdateStream extends Writable {
4739
4764
  objectMode: true
4740
4765
  });
4741
4766
  this.options = options;
4742
- this.batchSize = options.batchSize || 10;
4743
4767
  this.results = {
4744
4768
  successful: [],
4745
4769
  failed: [],
4746
4770
  totalProcessed: 0
4747
4771
  };
4748
- this.semaphore = new Sema(this.batchSize);
4772
+ this.semaphore = createPipelineBackpressureLock();
4749
4773
  }
4750
4774
  results;
4751
- batchSize;
4752
4775
  semaphore;
4753
4776
  async _write(chunk, _encoding, callback) {
4754
4777
  try {
@@ -4904,7 +4927,6 @@ runCmd.action(async (componentName, options, command) => {
4904
4927
  query,
4905
4928
  starts_with: startsWith
4906
4929
  },
4907
- batchSize: 12,
4908
4930
  onTotal: (total) => {
4909
4931
  storiesProgress.setTotal(total);
4910
4932
  migrationsProgress.setTotal(total);
@@ -4930,7 +4952,6 @@ runCmd.action(async (componentName, options, command) => {
4930
4952
  space,
4931
4953
  publish,
4932
4954
  dryRun,
4933
- batchSize: 12,
4934
4955
  onProgress: () => {
4935
4956
  updateProgress.increment();
4936
4957
  }
@@ -6412,7 +6433,8 @@ pushCmd$2.action(async (datasourceName, options, command) => {
6412
6433
  return;
6413
6434
  }
6414
6435
  } else if (filter) {
6415
- spaceState.local.datasources = spaceState.local.datasources.filter((datasource) => datasource.name.includes(filter));
6436
+ const isGlobPattern = /[*?[\]{}!]/.test(filter);
6437
+ spaceState.local.datasources = spaceState.local.datasources.filter((datasource) => isGlobPattern ? minimatch(datasource.name, filter) : datasource.name.includes(filter));
6416
6438
  if (!spaceState.local.datasources.length) {
6417
6439
  handleError(new CommandError(`No datasources found matching pattern "${filter}".`), verbose);
6418
6440
  return;
@@ -7109,6 +7131,41 @@ const fetchAssets = async ({ spaceId, params }) => {
7109
7131
  handleAPIError("pull_assets", toError(maybeError));
7110
7132
  }
7111
7133
  };
7134
+ const fetchAssetInternalTagsByName = async (spaceId) => {
7135
+ try {
7136
+ const client = getMapiClient();
7137
+ const tags = await fetchAllPages(
7138
+ (page) => client.internalTags.list({
7139
+ path: { space_id: Number(spaceId) },
7140
+ query: { page, by_object_type: "asset" },
7141
+ throwOnError: true
7142
+ }),
7143
+ (data) => data?.internal_tags ?? []
7144
+ );
7145
+ return new Map(
7146
+ tags.filter((tag) => typeof tag?.id === "number" && typeof tag?.name === "string").map((tag) => [tag.name, tag.id])
7147
+ );
7148
+ } catch (maybeError) {
7149
+ handleAPIError("pull_asset_internal_tags", toError(maybeError));
7150
+ }
7151
+ };
7152
+ const createAssetInternalTag = async (spaceId, name) => {
7153
+ try {
7154
+ const client = getMapiClient();
7155
+ const { data } = await client.internalTags.create({
7156
+ path: { space_id: Number(spaceId) },
7157
+ body: { name, object_type: "asset" },
7158
+ throwOnError: true
7159
+ });
7160
+ const tag = data?.internal_tag;
7161
+ if (typeof tag?.id !== "number" || typeof tag?.name !== "string") {
7162
+ throw new TypeError("Created internal tag is missing an id or name");
7163
+ }
7164
+ return { id: tag.id, name: tag.name };
7165
+ } catch (maybeError) {
7166
+ handleAPIError("push_asset_internal_tag", toError(maybeError), `Failed to create internal asset tag "${name}"`);
7167
+ }
7168
+ };
7112
7169
  const downloadFile = async (filename) => {
7113
7170
  const response = await fetch(filename);
7114
7171
  if (!response.ok) {
@@ -7312,8 +7369,55 @@ const getFolderFilename = (folder) => {
7312
7369
  const baseName = sanitizedName || folder.uuid;
7313
7370
  return `${baseName}_${folder.uuid}.json`;
7314
7371
  };
7372
+ const internalTagNamesFromAssets = (assets) => {
7373
+ const names = /* @__PURE__ */ new Set();
7374
+ for (const asset of assets) {
7375
+ for (const tag of asset.internal_tags_list ?? []) {
7376
+ if (typeof tag?.name === "string" && tag.name.length > 0) {
7377
+ names.add(tag.name);
7378
+ }
7379
+ }
7380
+ }
7381
+ return [...names];
7382
+ };
7383
+ const collectAssetInternalTagNames = async (directoryPath) => {
7384
+ const files = await readdir(directoryPath);
7385
+ const binaryFiles = files.filter((file) => SUPPORTED_ASSET_EXTENSIONS.has(extname(file).toLowerCase()));
7386
+ const sidecars = await Promise.all(
7387
+ binaryFiles.map((file) => loadSidecarAssetData(join(directoryPath, file)))
7388
+ );
7389
+ return internalTagNamesFromAssets(sidecars);
7390
+ };
7391
+ const ensureAssetInternalTags = async ({
7392
+ sourceTagNames,
7393
+ targetTagsByName,
7394
+ createTag,
7395
+ onTagCreated,
7396
+ onTagCreateError
7397
+ }) => {
7398
+ const tagsByName = new Map(targetTagsByName);
7399
+ const missing = [...new Set(sourceTagNames)].filter((name) => !tagsByName.has(name));
7400
+ for (const name of missing) {
7401
+ try {
7402
+ const created = await createTag(name);
7403
+ if (created) {
7404
+ tagsByName.set(created.name, created.id);
7405
+ onTagCreated?.(created.name);
7406
+ }
7407
+ } catch (maybeError) {
7408
+ onTagCreateError?.(name, toError(maybeError));
7409
+ }
7410
+ }
7411
+ return tagsByName;
7412
+ };
7315
7413
 
7316
- const apiConcurrencyLock$1 = new Sema(12);
7414
+ let _pipelineSlot$1 = null;
7415
+ const getPipelineSlot$1 = () => {
7416
+ if (!_pipelineSlot$1) {
7417
+ _pipelineSlot$1 = createPipelineBackpressureLock();
7418
+ }
7419
+ return _pipelineSlot$1;
7420
+ };
7317
7421
  const fetchAssetsStream = ({
7318
7422
  spaceId,
7319
7423
  params = {},
@@ -7370,7 +7474,7 @@ const downloadAssetStream = ({
7370
7474
  return new Transform({
7371
7475
  objectMode: true,
7372
7476
  async transform(asset, _encoding, callback) {
7373
- await apiConcurrencyLock$1.acquire();
7477
+ await getPipelineSlot$1().acquire();
7374
7478
  const task = downloadAssetFile(asset, { assetToken, region }).then((fileBuffer) => {
7375
7479
  if (!fileBuffer) {
7376
7480
  throw new Error("Invalid asset file!");
@@ -7381,7 +7485,7 @@ const downloadAssetStream = ({
7381
7485
  onAssetError?.(toError(maybeError), asset);
7382
7486
  }).finally(() => {
7383
7487
  onIncrement?.();
7384
- apiConcurrencyLock$1.release();
7488
+ getPipelineSlot$1().release();
7385
7489
  processing.delete(task);
7386
7490
  });
7387
7491
  processing.add(task);
@@ -7409,7 +7513,7 @@ const writeAssetStream = ({
7409
7513
  return new Writable({
7410
7514
  objectMode: true,
7411
7515
  async write(payload, _encoding, callback) {
7412
- await apiConcurrencyLock$1.acquire();
7516
+ await getPipelineSlot$1().acquire();
7413
7517
  const task = (async () => {
7414
7518
  try {
7415
7519
  await writeAsset(payload.asset, payload.fileBuffer);
@@ -7421,7 +7525,7 @@ const writeAssetStream = ({
7421
7525
  processing.add(task);
7422
7526
  task.finally(() => {
7423
7527
  onIncrement?.();
7424
- apiConcurrencyLock$1.release();
7528
+ getPipelineSlot$1().release();
7425
7529
  processing.delete(task);
7426
7530
  });
7427
7531
  callback();
@@ -7468,7 +7572,7 @@ const writeAssetFolderStream = ({
7468
7572
  return new Writable({
7469
7573
  objectMode: true,
7470
7574
  async write(folder, _encoding, callback) {
7471
- await apiConcurrencyLock$1.acquire();
7575
+ await getPipelineSlot$1().acquire();
7472
7576
  const task = (async () => {
7473
7577
  try {
7474
7578
  await writeAssetFolder(folder);
@@ -7480,7 +7584,7 @@ const writeAssetFolderStream = ({
7480
7584
  processing.add(task);
7481
7585
  task.finally(() => {
7482
7586
  onIncrement?.();
7483
- apiConcurrencyLock$1.release();
7587
+ getPipelineSlot$1().release();
7484
7588
  processing.delete(task);
7485
7589
  });
7486
7590
  callback();
@@ -7676,6 +7780,29 @@ const makeAppendAssetFolderManifestFSTransport = ({ manifestFile }) => async (lo
7676
7780
  created_at: createdAt
7677
7781
  }));
7678
7782
  };
7783
+ const mapInternalTagIds = (sourceIds, sourceTags, assetInternalTagsByName, onUnmappedTag) => {
7784
+ if (!sourceIds || sourceIds.length === 0) {
7785
+ return [];
7786
+ }
7787
+ const sourceNamesById = /* @__PURE__ */ new Map();
7788
+ for (const tag of sourceTags ?? []) {
7789
+ if (typeof tag?.id === "number" && typeof tag.name === "string") {
7790
+ sourceNamesById.set(tag.id, tag.name);
7791
+ }
7792
+ }
7793
+ const mapped = [];
7794
+ for (const raw of sourceIds) {
7795
+ const sourceId = Number(raw);
7796
+ const sourceName = sourceNamesById.get(sourceId);
7797
+ const targetId = typeof sourceName === "string" ? assetInternalTagsByName.get(sourceName) : void 0;
7798
+ if (typeof targetId === "number") {
7799
+ mapped.push(String(targetId));
7800
+ } else {
7801
+ onUnmappedTag?.({ sourceId, name: sourceName });
7802
+ }
7803
+ }
7804
+ return mapped;
7805
+ };
7679
7806
  const makeGetAssetAPITransport = ({ spaceId }) => async (assetId) => {
7680
7807
  const { data, response } = await getMapiClient().assets.get(assetId, {
7681
7808
  path: {
@@ -7714,11 +7841,14 @@ const processAsset = async ({
7714
7841
  assetBinaryPath,
7715
7842
  assetPath,
7716
7843
  transports,
7717
- maps
7844
+ maps,
7845
+ onUnmappedTag
7718
7846
  }) => {
7719
7847
  const remoteFolderId = localAsset.asset_folder_id && (maps.assetFolders.get(localAsset.asset_folder_id) || localAsset.asset_folder_id);
7720
7848
  const remoteAssetId = hasId(localAsset) ? maps.assets.get(localAsset.id)?.new.id || localAsset.id : void 0;
7721
7849
  const remoteAsset = remoteAssetId ? await transports.getAsset(remoteAssetId) : null;
7850
+ const sourceTags = localAsset.internal_tags_list;
7851
+ const resolveInternalTagIds = (sourceIds) => maps.assetInternalTagsByName ? mapInternalTagIds(sourceIds, sourceTags, maps.assetInternalTagsByName, onUnmappedTag) : (sourceIds ?? []).map((id) => String(id));
7722
7852
  let newRemoteAsset;
7723
7853
  let status;
7724
7854
  if (remoteAsset) {
@@ -7732,7 +7862,7 @@ const processAsset = async ({
7732
7862
  focus: "focus" in localAsset ? localAsset.focus : remoteAsset.focus,
7733
7863
  expire_at: "expire_at" in localAsset ? localAsset.expire_at : remoteAsset.expire_at,
7734
7864
  publish_at: "publish_at" in localAsset ? localAsset.publish_at : remoteAsset.publish_at,
7735
- internal_tag_ids: "internal_tag_ids" in localAsset ? localAsset.internal_tag_ids : remoteAsset.internal_tag_ids,
7865
+ internal_tag_ids: "internal_tag_ids" in localAsset ? resolveInternalTagIds(localAsset.internal_tag_ids) : remoteAsset.internal_tag_ids,
7736
7866
  meta_data: "meta_data" in localAsset ? localAsset.meta_data : remoteAsset.meta_data
7737
7867
  };
7738
7868
  await transports.updateAsset(
@@ -7743,9 +7873,12 @@ const processAsset = async ({
7743
7873
  newRemoteAsset = { ...remoteAsset, ...updatePayload };
7744
7874
  status = "updated";
7745
7875
  } else if (hasShortFilename(localAsset)) {
7876
+ const { internal_tags_list: _internalTagsList, internal_tag_ids: _sourceTagIds, ...rest } = localAsset;
7877
+ const mappedTagIds = "internal_tag_ids" in localAsset ? resolveInternalTagIds(localAsset.internal_tag_ids) : void 0;
7746
7878
  const createPayload = {
7747
- ...localAsset,
7748
- asset_folder_id: remoteFolderId
7879
+ ...rest,
7880
+ asset_folder_id: remoteFolderId,
7881
+ ...mappedTagIds !== void 0 ? { internal_tag_ids: mappedTagIds } : {}
7749
7882
  };
7750
7883
  newRemoteAsset = await transports.createAsset(createPayload, fileBuffer);
7751
7884
  status = "created";
@@ -7763,13 +7896,14 @@ const upsertAssetStream = ({
7763
7896
  maps,
7764
7897
  onIncrement,
7765
7898
  onAssetSuccess,
7766
- onAssetError
7899
+ onAssetError,
7900
+ onUnmappedTag
7767
7901
  }) => {
7768
7902
  const processing = /* @__PURE__ */ new Set();
7769
7903
  return new Writable({
7770
7904
  objectMode: true,
7771
7905
  async write({ asset: localAsset, context }, _encoding, callback) {
7772
- await apiConcurrencyLock$1.acquire();
7906
+ await getPipelineSlot$1().acquire();
7773
7907
  const task = (async () => {
7774
7908
  try {
7775
7909
  const { remoteAsset } = await processAsset({
@@ -7778,7 +7912,8 @@ const upsertAssetStream = ({
7778
7912
  assetBinaryPath: context.assetBinaryPath,
7779
7913
  assetPath: context.assetPath,
7780
7914
  transports,
7781
- maps
7915
+ maps,
7916
+ onUnmappedTag
7782
7917
  });
7783
7918
  onAssetSuccess?.(localAsset, remoteAsset);
7784
7919
  } catch (maybeError) {
@@ -7788,7 +7923,7 @@ const upsertAssetStream = ({
7788
7923
  processing.add(task);
7789
7924
  task.finally(() => {
7790
7925
  onIncrement?.();
7791
- apiConcurrencyLock$1.release();
7926
+ getPipelineSlot$1().release();
7792
7927
  processing.delete(task);
7793
7928
  });
7794
7929
  callback();
@@ -8111,7 +8246,13 @@ const storyRefMapper = (story, { schemas, maps }) => {
8111
8246
  };
8112
8247
  };
8113
8248
 
8114
- const apiConcurrencyLock = new Sema(12);
8249
+ let _pipelineSlot = null;
8250
+ const getPipelineSlot = () => {
8251
+ if (!_pipelineSlot) {
8252
+ _pipelineSlot = createPipelineBackpressureLock();
8253
+ }
8254
+ return _pipelineSlot;
8255
+ };
8115
8256
  const fetchStoriesStream = ({
8116
8257
  spaceId,
8117
8258
  params = {},
@@ -8167,7 +8308,7 @@ const fetchStoryStream = ({
8167
8308
  return new Transform({
8168
8309
  objectMode: true,
8169
8310
  async transform(listStory, _encoding, callback) {
8170
- await apiConcurrencyLock.acquire();
8311
+ await getPipelineSlot().acquire();
8171
8312
  const task = fetchStory(spaceId, listStory.id.toString()).then((story) => {
8172
8313
  if (typeof story === "undefined") {
8173
8314
  throw new TypeError("Invalid story!");
@@ -8178,7 +8319,7 @@ const fetchStoryStream = ({
8178
8319
  onStoryError?.(toError(maybeError), listStory);
8179
8320
  }).finally(() => {
8180
8321
  onIncrement?.();
8181
- apiConcurrencyLock.release();
8322
+ getPipelineSlot().release();
8182
8323
  processing.delete(task);
8183
8324
  });
8184
8325
  processing.add(task);
@@ -8347,7 +8488,7 @@ const createStoriesForLevel = async ({
8347
8488
  onStoryError
8348
8489
  }) => {
8349
8490
  const processEntry = async (entry) => {
8350
- await apiConcurrencyLock.acquire();
8491
+ await getPipelineSlot().acquire();
8351
8492
  try {
8352
8493
  const mappedStoryId = maps.stories?.get(entry.id);
8353
8494
  const mappedRemoteStory = mappedStoryId ? existingTargetStories.byId.get(Number(mappedStoryId)) : void 0;
@@ -8394,7 +8535,7 @@ const createStoriesForLevel = async ({
8394
8535
  } catch (maybeError) {
8395
8536
  onStoryError?.(toError(maybeError), entry);
8396
8537
  } finally {
8397
- apiConcurrencyLock.release();
8538
+ getPipelineSlot().release();
8398
8539
  }
8399
8540
  };
8400
8541
  const folders = level.filter((e) => e.is_folder);
@@ -8451,7 +8592,7 @@ const writeStoryStream = ({
8451
8592
  return new Writable({
8452
8593
  objectMode: true,
8453
8594
  async write(mappedLocalStory, _encoding, callback) {
8454
- await apiConcurrencyLock.acquire();
8595
+ await getPipelineSlot().acquire();
8455
8596
  const task = (async () => {
8456
8597
  try {
8457
8598
  const remoteStory = await transports.writeStory(mappedLocalStory);
@@ -8464,7 +8605,7 @@ const writeStoryStream = ({
8464
8605
  processing.add(task);
8465
8606
  task.finally(() => {
8466
8607
  onIncrement?.();
8467
- apiConcurrencyLock.release();
8608
+ getPipelineSlot().release();
8468
8609
  processing.delete(task);
8469
8610
  });
8470
8611
  callback();
@@ -8677,7 +8818,8 @@ const upsertAssetsPipeline = async ({
8677
8818
  logger,
8678
8819
  maps,
8679
8820
  transports,
8680
- ui
8821
+ ui,
8822
+ onUnmappedTag
8681
8823
  }) => {
8682
8824
  const assetProgress = ui.createProgressBar({ title: "Assets...".padEnd(PROGRESS_BAR_PADDING) });
8683
8825
  const summary = { total: 0, succeeded: 0, failed: 0 };
@@ -8711,6 +8853,7 @@ const upsertAssetsPipeline = async ({
8711
8853
  steps.push(upsertAssetStream({
8712
8854
  transports,
8713
8855
  maps,
8856
+ onUnmappedTag,
8714
8857
  onIncrement: () => assetProgress.increment(),
8715
8858
  onAssetSuccess: (localAssetResult, remoteAsset) => {
8716
8859
  if ("id" in localAssetResult && localAssetResult.id) {
@@ -8930,12 +9073,34 @@ pushCmd$1.action(async (assetInput, options, command) => {
8930
9073
  } : makeUpdateAssetAPITransport({ spaceId: targetSpace });
8931
9074
  const assetManifestTransport = options.dryRun ? () => Promise.resolve() : makeAppendAssetManifestFSTransport({ manifestFile });
8932
9075
  const cleanupAssetTransport = options.cleanup && !options.dryRun ? makeCleanupAssetFSTransport() : () => Promise.resolve();
9076
+ let assetInternalTagsByName = fromSpace === targetSpace ? void 0 : await fetchAssetInternalTagsByName(targetSpace);
9077
+ if (assetInternalTagsByName && !options.dryRun) {
9078
+ const sourceTagNames = assetBinaryPath ? internalTagNamesFromAssets(assetData ? [assetData] : []) : await collectAssetInternalTagNames(assetsDirectoryPath);
9079
+ const createdTagLabels = [];
9080
+ assetInternalTagsByName = await ensureAssetInternalTags({
9081
+ sourceTagNames,
9082
+ targetTagsByName: assetInternalTagsByName,
9083
+ createTag: (name) => createAssetInternalTag(targetSpace, name),
9084
+ onTagCreated: (name) => createdTagLabels.push(name),
9085
+ onTagCreateError: (name, error) => logger.warn(`Failed to create internal asset tag "${name}"`, { error: error.message })
9086
+ });
9087
+ if (createdTagLabels.length > 0) {
9088
+ const labels = [...createdTagLabels].sort((a, b) => a.localeCompare(b));
9089
+ const message = `Created ${labels.length} internal asset tag${labels.length === 1 ? "" : "s"} in target space: ${labels.join(", ")}`;
9090
+ ui.info(message);
9091
+ logger.info(message, { createdTags: labels });
9092
+ }
9093
+ }
9094
+ const unmappedTagLabels = /* @__PURE__ */ new Set();
9095
+ const onUnmappedTag = ({ sourceId, name }) => {
9096
+ unmappedTagLabels.add(name ?? `#${sourceId}`);
9097
+ };
8933
9098
  summaries.push(...await upsertAssetsPipeline({
8934
9099
  assetBinaryPath,
8935
9100
  assetData,
8936
9101
  directoryPath: assetsDirectoryPath,
8937
9102
  logger,
8938
- maps,
9103
+ maps: { ...maps, assetInternalTagsByName },
8939
9104
  transports: {
8940
9105
  getAsset: getAssetTransport,
8941
9106
  createAsset: createAssetTransport,
@@ -8943,8 +9108,15 @@ pushCmd$1.action(async (assetInput, options, command) => {
8943
9108
  appendAssetManifest: assetManifestTransport,
8944
9109
  cleanupAsset: cleanupAssetTransport
8945
9110
  },
8946
- ui
9111
+ ui,
9112
+ onUnmappedTag
8947
9113
  }));
9114
+ if (unmappedTagLabels.size > 0) {
9115
+ const labels = [...unmappedTagLabels].sort((a, b) => a.localeCompare(b));
9116
+ const message = `Dropped ${labels.length} unknown internal asset tag${labels.length === 1 ? "" : "s"} not present in target space: ${labels.join(", ")}`;
9117
+ ui.warn(message);
9118
+ logger.warn(message, { unmappedTags: labels });
9119
+ }
8948
9120
  const hasUpdatedFilename = (entry) => "filename" in entry.old && entry.old.filename !== entry.new.filename;
8949
9121
  const hasMetadata = (entry) => "meta_data" in entry.new && entry.new.meta_data;
8950
9122
  const hasUpdatedAssets = maps.assets.values().some((v) => hasUpdatedFilename(v) || hasMetadata(v));