ugcinc 4.22.0 → 4.24.0

package/dist/media-analysis.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.mediaAnalysisResultSchema = exports.videoTextResultSchema = exports.imageTextResultSchema = exports.textLineSchema = exports.textRegionSchema = exports.readVideoTextSchema = exports.readImageTextSchema = exports.scenesResultSchema = exports.framesResultSchema = exports.analysisSourceSchema = exports.mediaAnalysisStatusSchema = exports.extractFramesSchema = exports.detectScenesSchema = void 0;
+exports.mediaAnalysisResultSchema = exports.measureLayoutResultSchema = exports.sampleColorsResultSchema = exports.measureLayoutSchema = exports.sampleColorsSchema = exports.binaryMaskSchema = exports.audioTranscriptResultSchema = exports.audioLevelsResultSchema = exports.measureAudioLevelsSchema = exports.transcribeAudioSchema = exports.videoTextResultSchema = exports.imageTextResultSchema = exports.textLineSchema = exports.textRegionSchema = exports.readVideoTextSchema = exports.readImageTextSchema = exports.scenesResultSchema = exports.framesResultSchema = exports.analysisSourceSchema = exports.mediaAnalysisStatusSchema = exports.extractFramesSchema = exports.detectScenesSchema = void 0;
 const zod_1 = require("zod");
 const videoUrl = zod_1.z.string().url().max(8192).refine(value => {
     const url = new URL(value);
@@ -62,4 +62,48 @@ exports.videoTextResultSchema = zod_1.z.object({ version: zod_1.z.literal(1), op
     coverage: zod_1.z.object({ mode: zod_1.z.enum(['fast', 'frame_scan']), range: intervalSchema, complete: zod_1.z.boolean(), uninspected: zod_1.z.array(intervalSchema),
         inspected: zod_1.z.array(intervalSchema.extend({ frameIndex: zod_1.z.number().int(), sourcePts: zod_1.z.string() })) }),
 });
-exports.mediaAnalysisResultSchema = zod_1.z.discriminatedUnion('operation', [exports.framesResultSchema, exports.scenesResultSchema, exports.imageTextResultSchema, exports.videoTextResultSchema]);
+const audioRange = zod_1.z.object({ startMs: time, endMs: time }).strict().refine(v => v.endMs > v.startMs && v.endMs - v.startMs <= 120000, 'Audio range must be positive and at most 120 seconds');
+exports.transcribeAudioSchema = zod_1.z.object({ url: videoUrl, range: audioRange.optional(),
+    language: zod_1.z.enum(['auto', 'en', 'es', 'fr', 'de', 'it', 'pt', 'ja', 'ko', 'zh', 'ar', 'hi', 'ru']).default('auto'),
+    timing: zod_1.z.enum(['word', 'segment']).default('word'), channel: zod_1.z.number().int().min(0).max(1).optional() }).strict();
+exports.measureAudioLevelsSchema = zod_1.z.object({ url: videoUrl, range: audioRange.optional(),
+    windowMs: zod_1.z.number().finite().min(20).max(5000).default(100), silenceThresholdDbfs: zod_1.z.number().finite().min(-100).max(-10).default(-50) }).strict();
+const audioSourceSchema = zod_1.z.object({ sha256: zod_1.z.string(), bytes: zod_1.z.number(), hasAudio: zod_1.z.boolean(), durationMs: time, originStartMs: zod_1.z.number(),
+    audioStartPts: zod_1.z.string().nullable(), audioTimeBase: zod_1.z.string().nullable(), sampleRate: zod_1.z.number().nullable(), channels: zod_1.z.number().nullable(), channelLayout: zod_1.z.string().nullable(), audioOffsetMs: zod_1.z.number().nullable() });
+const audioCoverageSchema = zod_1.z.object({ requestedRange: intervalSchema, analyzed: zod_1.z.array(intervalSchema), withoutAudio: zod_1.z.array(intervalSchema), sampleRoundingToleranceMs: time });
+const dbValue = zod_1.z.number().finite().nullable();
+const levelChannelSchema = zod_1.z.object({ index: zod_1.z.number().int(), rmsDbfs: dbValue, samplePeakDbfs: dbValue, potentialClippedSamples: zod_1.z.number().int().nonnegative() });
+exports.audioLevelsResultSchema = zod_1.z.object({ version: zod_1.z.literal(1), operation: zod_1.z.literal('measure_audio_levels'), source: audioSourceSchema,
+    status: zod_1.z.enum(['no_audio', 'measured']), coverage: audioCoverageSchema,
+    channels: zod_1.z.array(levelChannelSchema.extend({ truePeakDbtp: dbValue, sampleCount: zod_1.z.number().int(), intersampleOvershoot: zod_1.z.boolean() })),
+    windows: zod_1.z.array(intervalSchema.extend({ channels: zod_1.z.array(levelChannelSchema) })), silence: zod_1.z.array(intervalSchema), integratedLufs: dbValue, truePeakDbtp: dbValue,
+    loudnessTimeline: zod_1.z.array(intervalSchema.extend({ momentaryLufs: dbValue, shortTermLufs: dbValue })),
+    settings: zod_1.z.object({ windowMs: zod_1.z.number(), silenceThresholdDbfs: zod_1.z.number(), silenceMethod: zod_1.z.literal('all-channel-window-rms'), potentialClippingThreshold: zod_1.z.number(), channelPolicy: zod_1.z.literal('native-separate'), loudnessMethod: zod_1.z.literal('FFmpeg-ebur128-native-channels'), ffmpegVersion: zod_1.z.string() }),
+    warnings: zod_1.z.array(zod_1.z.string()), previewUrl: zod_1.z.string().url().optional(), manifestUrl: zod_1.z.string().url() });
+const speechModelSchema = zod_1.z.object({ name: zod_1.z.string(), repository: zod_1.z.string(), revision: zod_1.z.string(), sha256: zod_1.z.record(zod_1.z.string()), runtime: zod_1.z.string(), device: zod_1.z.literal('cpu'), computeType: zod_1.z.literal('int8'), vad: zod_1.z.string(), beamSize: zod_1.z.number(), temperature: zod_1.z.number(), conditionOnPreviousText: zod_1.z.boolean() });
+const estimatedWord = zod_1.z.object({ text: zod_1.z.string(), startMs: zod_1.z.number().finite(), endMs: zod_1.z.number().finite(), timing: zod_1.z.literal('estimated'), probability: zod_1.z.number() });
+exports.audioTranscriptResultSchema = zod_1.z.object({ version: zod_1.z.literal(1), operation: zod_1.z.literal('transcribe_audio'), source: audioSourceSchema,
+    status: zod_1.z.enum(['no_audio', 'no_speech_detected', 'transcribed']), text: zod_1.z.string(), coverage: audioCoverageSchema, model: speechModelSchema,
+    segments: zod_1.z.array(zod_1.z.object({ index: zod_1.z.number().int(), text: zod_1.z.string(), startMs: zod_1.z.number().finite(), endMs: zod_1.z.number().finite(), timing: zod_1.z.literal('estimated'), avgLogProbability: zod_1.z.number(), noSpeechProbability: zod_1.z.number(), words: zod_1.z.array(estimatedWord) })),
+    language: zod_1.z.string().nullable(), languageProbability: zod_1.z.number().nullable(), segmentTiming: zod_1.z.enum(['estimated', 'unavailable']), wordTiming: zod_1.z.enum(['estimated', 'unavailable']), speakerLabels: zod_1.z.literal('unavailable'),
+    channelPolicy: zod_1.z.string(), warnings: zod_1.z.array(zod_1.z.string()), audioPreviewUrl: zod_1.z.string().url().optional(), manifestUrl: zod_1.z.string().url() });
+const imageBox = zod_1.z.object({ x: zod_1.z.number().finite().min(0).max(4096), y: zod_1.z.number().finite().min(0).max(4096), width: zod_1.z.number().finite().positive().max(4096), height: zod_1.z.number().finite().positive().max(4096) }).strict().refine(b => b.x + b.width <= 4096 && b.y + b.height <= 4096, 'Box exceeds image limit');
+exports.binaryMaskSchema = zod_1.z.object({ width: zod_1.z.number().int().min(1).max(4096), height: zod_1.z.number().int().min(1).max(4096), counts: zod_1.z.array(zod_1.z.number().int().min(0).max(16777216)).min(1).max(100000) }).strict().refine(m => m.counts.reduce((s, n) => s + n, 0) === m.width * m.height && m.counts.every((n, i) => i === 0 || n > 0), 'Row-major runs must sum to width*height and alternate background/foreground; only first may be zero');
+exports.sampleColorsSchema = zod_1.z.object({ url: videoUrl,
+    point: zod_1.z.object({ x: zod_1.z.number().int().min(0).max(4095), y: zod_1.z.number().int().min(0).max(4095) }).strict().optional(),
+    box: imageBox.optional(), mask: exports.binaryMaskSchema.optional(), maxColors: zod_1.z.number().int().min(1).max(32).default(8),
+}).strict().refine(v => [v.point, v.box, v.mask].filter(x => x !== undefined).length <= 1, 'Provide at most one point, box or mask');
+const layoutElement = zod_1.z.object({ id: zod_1.z.string().min(1).max(80), box: imageBox.optional(), mask: exports.binaryMaskSchema.optional() }).strict().refine(v => Boolean(v.box) !== Boolean(v.mask), 'Provide box or mask exclusively');
+exports.measureLayoutSchema = zod_1.z.object({ url: videoUrl, elements: zod_1.z.array(layoutElement).min(1).max(32) }).strict().refine(v => new Set(v.elements.map(e => e.id)).size === v.elements.length && v.elements.reduce((s, e) => s + (e.mask?.counts.length ?? 0), 0) <= 100000, 'IDs must be unique; total mask runs at most 100000');
+const visualSource = zod_1.z.object({ encodedWidth: zod_1.z.number(), encodedHeight: zod_1.z.number(), displayWidth: zod_1.z.number(), displayHeight: zod_1.z.number(), exifOrientation: zod_1.z.number(), colorSpace: zod_1.z.literal('sRGB'), colorPolicy: zod_1.z.enum(['embedded-ICC-to-sRGB-relative-colorimetric', 'untagged-assumed-sRGB']), alphaPolicy: zod_1.z.literal('straight-alpha-no-background-composite'), sha256: zod_1.z.string(), bytes: zod_1.z.number() });
+const finiteNumber = zod_1.z.number().finite();
+const edges = zod_1.z.object({ left: finiteNumber, top: finiteNumber, right: finiteNumber, bottom: finiteNumber });
+exports.sampleColorsResultSchema = zod_1.z.object({ version: zod_1.z.literal(1), operation: zod_1.z.literal('sample_colors'), source: visualSource,
+    point: zod_1.z.object({ x: finiteNumber, y: finiteNumber, rgba: zod_1.z.array(zod_1.z.number().int().min(0).max(255)).length(4) }).nullable(), selectionBox: boxSchema.nullable(),
+    selectedPixelCount: time, visiblePixelCount: time, alphaWeight: time, palette: zod_1.z.array(zod_1.z.object({ rgb: zod_1.z.array(zod_1.z.number().int().min(0).max(255)).length(3), hex: zod_1.z.string(), proportion: zod_1.z.number().min(0).max(1) })), otherProportion: zod_1.z.number().min(0).max(1),
+    method: zod_1.z.enum(['exact-observed-colors', '5-bit-sRGB-bins-alpha-weighted-mean']), warnings: zod_1.z.array(zod_1.z.string()), previewUrl: zod_1.z.string().url(), manifestUrl: zod_1.z.string().url() });
+exports.measureLayoutResultSchema = zod_1.z.object({ version: zod_1.z.literal(1), operation: zod_1.z.literal('measure_layout'), source: visualSource, coordinates: zod_1.z.literal('display-oriented-source-pixels'),
+    elements: zod_1.z.array(zod_1.z.object({ id: zod_1.z.string(), geometry: zod_1.z.enum(['box', 'mask']), box: boxSchema.nullable(), normalizedBox: boxSchema.nullable(), boxArea: time, visibleArea: time.nullable(), visibleFractionOfCanvas: time.nullable(), boxCenter: pointSchema.nullable(), visibleCentroid: pointSchema.nullable(), margins: edges.nullable() })),
+    pairs: zod_1.z.array(zod_1.z.object({ a: zod_1.z.string(), b: zod_1.z.string(), boxIntersectionArea: time, boxIoU: time.nullable(), visibleIntersectionArea: time.nullable(), visibleIoU: time.nullable(), axisGaps: zod_1.z.object({ x: finiteNumber, y: finiteNumber }).nullable(), alignmentDeltas: edges.extend({ centerX: finiteNumber, centerY: finiteNumber }).nullable() })),
+    warnings: zod_1.z.array(zod_1.z.string()), previewUrl: zod_1.z.string().url(), manifestUrl: zod_1.z.string().url() });
+exports.mediaAnalysisResultSchema = zod_1.z.discriminatedUnion('operation', [exports.framesResultSchema, exports.scenesResultSchema, exports.imageTextResultSchema, exports.videoTextResultSchema, exports.audioTranscriptResultSchema, exports.audioLevelsResultSchema, exports.sampleColorsResultSchema, exports.measureLayoutResultSchema]);

package/dist/media.d.ts

@@ -1,5 +1,5 @@
 import { BaseClient } from './base';
-import type { ExtractFramesParams, DetectScenesParams, ReadImageTextParams, ReadVideoTextParams, MediaAnalysisJob } from './media-analysis';
+import type { SampleColorsParams, MeasureLayoutParams, TranscribeAudioParams, MeasureAudioLevelsParams, ExtractFramesParams, DetectScenesParams, ReadImageTextParams, ReadVideoTextParams, MediaAnalysisJob } from './media-analysis';
 import type { ListProfilePostsParams, ListProfilePostsResult, DownloadSocialPostParams, DownloadSocialPostResult } from './social-acquisition';
 import type { ApiResponse } from './types';
 export interface UserMedia {
@@ -200,6 +200,12 @@ export interface OwnedUploadResponse {
  * Client for managing media files
  */
 export declare class MediaClient extends BaseClient {
+    /** Transcribe locally with explicit estimated segment/word timing. */
+    sampleColors(params: SampleColorsParams): Promise<ApiResponse<MediaAnalysisJob>>;
+    measureLayout(params: MeasureLayoutParams): Promise<ApiResponse<MediaAnalysisJob>>;
+    transcribeAudio(params: TranscribeAudioParams): Promise<ApiResponse<MediaAnalysisJob>>;
+    /** Measure native-channel loudness, RMS/peaks, silence and potential clipping. */
+    measureAudioLevels(params: MeasureAudioLevelsParams): Promise<ApiResponse<MediaAnalysisJob>>;
     /** Read English text with source-coordinate polygons and an annotated preview. */
     readImageText(params: ReadImageTextParams): Promise<ApiResponse<MediaAnalysisJob>>;
     readVideoText(params: ReadVideoTextParams): Promise<ApiResponse<MediaAnalysisJob>>;

package/dist/media.js

@@ -6,6 +6,20 @@ const base_1 = require("./base");
  * Client for managing media files
  */
 class MediaClient extends base_1.BaseClient {
+    /** Transcribe locally with explicit estimated segment/word timing. */
+    async sampleColors(params) {
+        return this.waitForAnalysis(await this.post('/media/analysis/sample-colors', params));
+    }
+    async measureLayout(params) {
+        return this.waitForAnalysis(await this.post('/media/analysis/measure-layout', params));
+    }
+    async transcribeAudio(params) {
+        return this.waitForAnalysis(await this.post('/media/analysis/transcribe-audio', params));
+    }
+    /** Measure native-channel loudness, RMS/peaks, silence and potential clipping. */
+    async measureAudioLevels(params) {
+        return this.waitForAnalysis(await this.post('/media/analysis/measure-audio-levels', params));
+    }
     /** Read English text with source-coordinate polygons and an annotated preview. */
     async readImageText(params) {
         return this.waitForAnalysis(await this.post('/media/analysis/read-image-text', params));

package/dist/tools/media.js

@@ -4,6 +4,10 @@ exports.mediaTools = void 0;
 const media_analysis_1 = require("../media-analysis");
 const zod_1 = require("zod");
 exports.mediaTools = [
+    { name: 'sample_colors', description: 'Sample observed sRGB colors from an image URL: optional integer point, half-open source-pixel box, or full-size row-major binary RLE mask. Counts alternate background/foreground starting with background (not COCO order). Default whole image, maxColors 8 (1–32). Returns RGBA point and alpha-weighted palette, method, omitted proportion, source hash and preview. Does not recover original authoring colors. Use get_media_analysis if pending.', schema: media_analysis_1.sampleColorsSchema, execute: async (client, params) => client.media.sampleColors(params) },
+    { name: 'measure_layout', description: 'Measure supplied element boxes or binary masks against an image URL. Returns margins, centers, spacing, alignment deltas and overlap. Coordinates are display-oriented source pixels. Masks match source dimensions, row-major counts alternate background/foreground starting with background (not COCO). Up to 32 uniquely identified elements. Visible area/intersection requires masks; boxes do not imply visible ink. Does not detect elements. Use get_media_analysis if pending.', schema: media_analysis_1.measureLayoutSchema, execute: async (client, params) => client.media.measureLayout(params) },
+    { name: 'transcribe_audio', description: 'Transcribe audio/video using an owned local speech model. Returns text, source-relative milliseconds, estimated segment/word timings, model details and an audio preview. No forced alignment or speaker separation. Optional range <=120 seconds and channel 0/1; default averages channels. If pending, use get_media_analysis.', schema: media_analysis_1.transcribeAudioSchema, execute: async (client, params) => client.media.transcribeAudio(params) },
+    { name: 'measure_audio_levels', description: 'Measure audio/video native-channel RMS and sample peaks, integrated/momentary/short-term loudness, estimated true peaks, window-based silence and possible clipping. Distinguishes missing audio from silence and sample peaks from intersample overshoot. Optional range <=120 seconds; default windows 100ms. If pending, use get_media_analysis.', schema: media_analysis_1.measureAudioLevelsSchema, execute: async (client, params) => client.media.measureAudioLevels(params) },
     { name: 'read_image_text', description: 'Read English image text with raw recognition, polygons in display-oriented source pixels, inferred lines/order and confidence. Includes annotated preview. Empty output does not prove no text. If pending, use get_media_analysis.', schema: media_analysis_1.readImageTextSchema, execute: async (client, params) => client.media.readImageText(params) },
     { name: 'read_video_text', description: 'Read English video text and positional changes. fast samples frames (default 1000ms); frame_scan inspects every decoded frame in range. Returns source PTS, observed text tracks and explicit uninspected gaps. At most 240 inspected frames and 120 seconds per request. Sampling cannot guarantee exact appearance/disappearance. If pending, use get_media_analysis.', schema: media_analysis_1.readVideoTextSchema, execute: async (client, params) => client.media.readVideoText(params) },
     { name: 'extract_frames', description: 'Extract video frames at timestampsMs or a range {startMs,endMs,intervalMs}; milliseconds relative to the first displayed frame. Returns actual decoded times and optional contact sheet. At most 48 samples. If pending, use get_media_analysis with jobId.', schema: media_analysis_1.extractFramesSchema, execute: async (client, params) => client.media.extractFrames(params) },

package/dist/types.d.ts

@@ -20,4 +20,4 @@ export interface ErrorResponse {
 export type ApiResponse<T> = SuccessResponse<T> | ErrorResponse;
 export type { PublicPostFailureCode, PublicPostRecovery, PublicPostNextAction, PublicPostFailure } from './post-failure';
 export type { ListProfilePostsParams, ListProfilePostsResult, DownloadSocialPostParams, DownloadSocialPostResult, SocialReferencePost, SocialReferenceAsset } from './social-acquisition';
-export type { ReadImageTextParams, ReadVideoTextParams, ImageTextResult, VideoTextResult, ExtractFramesParams, DetectScenesParams, FramesResult, ScenesResult, MediaAnalysisResult, MediaAnalysisJob } from "./media-analysis";
+export type { SampleColorsParams, MeasureLayoutParams, SampleColorsResult, MeasureLayoutResult, TranscribeAudioParams, MeasureAudioLevelsParams, AudioTranscriptResult, AudioLevelsResult, ReadImageTextParams, ReadVideoTextParams, ImageTextResult, VideoTextResult, ExtractFramesParams, DetectScenesParams, FramesResult, ScenesResult, MediaAnalysisResult, MediaAnalysisJob } from "./media-analysis";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ugcinc",
-  "version": "4.22.0",
+  "version": "4.24.0",
   "description": "TypeScript/JavaScript client for the UGC Inc API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",