youtube-transcript-plus 1.0.4 → 1.1.1

package/README.md

@@ -58,19 +58,37 @@ fetchTranscript('videoId_or_URL', {
 
 ### Custom Fetch Functions
 
-You can inject custom `videoFetch` and `transcriptFetch` functions to modify the fetch behavior, such as using a proxy or custom headers.
+You can inject custom `videoFetch`, `playerFetch`, and `transcriptFetch` functions to modify the fetch behavior, such as using a proxy or custom headers. The library makes three types of HTTP requests:
+
+1. **`videoFetch`**: Fetches the YouTube video page (GET request)
+2. **`playerFetch`**: Calls YouTube's Innertube API to get caption tracks (POST request)
+3. **`transcriptFetch`**: Downloads the actual transcript data (GET request)
 
 ```javascript
 fetchTranscript('videoId_or_URL', {
   videoFetch: async ({ url, lang, userAgent }) => {
+    // Custom logic for video page fetch (GET)
+    return fetch(`https://my-proxy-server.com/?url=${encodeURIComponent(url)}`, {
+      headers: {
+        ...(lang && { 'Accept-Language': lang }),
+        'User-Agent': userAgent,
+      },
+    });
+  },
+  playerFetch: async ({ url, method, body, headers, lang, userAgent }) => {
+    // Custom logic for Innertube API call (POST)
     return fetch(`https://my-proxy-server.com/?url=${encodeURIComponent(url)}`, {
+      method,
       headers: {
         ...(lang && { 'Accept-Language': lang }),
         'User-Agent': userAgent,
+        ...headers,
       },
+      body,
     });
   },
   transcriptFetch: async ({ url, lang, userAgent }) => {
+    // Custom logic for transcript data fetch (GET)
     return fetch(`https://my-proxy-server.com/?url=${encodeURIComponent(url)}`, {
       headers: {
         ...(lang && { 'Accept-Language': lang }),
@@ -187,6 +205,7 @@ The repository includes several example files in the `example/` directory to dem
 3. **`fs-caching-usage.js`**: Demonstrates how to use the `FsCache` to cache transcripts on the file system with a 1-day TTL.
 4. **`language-usage.js`**: Shows how to fetch a transcript in a specific language (e.g., French).
 5. **`proxy-usage.js`**: Demonstrates how to use a proxy server to fetch transcripts, which can be useful for bypassing rate limits or accessing restricted content.
+6. **`custom-fetch-usage.js`**: Shows how to use all three custom fetch functions (`videoFetch`, `playerFetch`, `transcriptFetch`) with logging and custom headers.
 
 These examples can be found in the `example/` directory of the repository.
 
@@ -203,8 +222,9 @@ Fetches the transcript for a YouTube video.
   - **`cache`**: Custom caching strategy.
   - **`cacheTTL`**: Time-to-live for cache entries in milliseconds.
   - **`disableHttps`**: Set to `true` to use HTTP instead of HTTPS for YouTube requests.
-  - **`videoFetch`**: Custom fetch function for the video page request.
-  - **`transcriptFetch`**: Custom fetch function for the transcript request.
+  - **`videoFetch`**: Custom fetch function for the video page request (GET).
+  - **`playerFetch`**: Custom fetch function for the YouTube Innertube API request (POST).
+  - **`transcriptFetch`**: Custom fetch function for the transcript data request (GET).
 
 Returns a `Promise<TranscriptResponse[]>` where each item in the array represents a transcript segment with the following properties:
 

package/dist/constants.d.ts

@@ -1,3 +1,4 @@
 export declare const DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
 export declare const RE_YOUTUBE: RegExp;
 export declare const RE_XML_TRANSCRIPT: RegExp;
+export declare const DEFAULT_CACHE_TTL = 3600000;

package/dist/index.d.ts

@@ -1,9 +1,13 @@
 import { TranscriptConfig, TranscriptResponse } from './types';
+/**
+ * Implementation notes:
+ * - Keeps the public surface identical.
+ * - Internals now use YouTube Innertube `player` to discover captionTracks instead of scraping the watch HTML.
+ * - Honors `lang`, custom fetch hooks (`videoFetch`, `transcriptFetch`), and optional cache strategy.
+ */
 export declare class YoutubeTranscript {
     private config?;
-    constructor(config?: TranscriptConfig & {
-        cacheTTL?: number;
-    });
+    constructor(config?: TranscriptConfig);
     fetchTranscript(videoId: string): Promise<TranscriptResponse[]>;
     static fetchTranscript(videoId: string, config?: TranscriptConfig): Promise<TranscriptResponse[]>;
 }

package/dist/types.d.ts

@@ -2,22 +2,23 @@ export interface CacheStrategy {
     get(key: string): Promise<string | null>;
     set(key: string, value: string, ttl?: number): Promise<void>;
 }
+export interface FetchParams {
+    url: string;
+    lang?: string;
+    userAgent?: string;
+    method?: 'GET' | 'POST';
+    body?: string;
+    headers?: Record<string, string>;
+}
 export interface TranscriptConfig {
     lang?: string;
     userAgent?: string;
     cache?: CacheStrategy;
     cacheTTL?: number;
     disableHttps?: boolean;
-    videoFetch?: (params: {
-        url: string;
-        lang?: string;
-        userAgent?: string;
-    }) => Promise<Response>;
-    transcriptFetch?: (params: {
-        url: string;
-        lang?: string;
-        userAgent?: string;
-    }) => Promise<Response>;
+    videoFetch?: (params: FetchParams) => Promise<Response>;
+    transcriptFetch?: (params: FetchParams) => Promise<Response>;
+    playerFetch?: (params: FetchParams) => Promise<Response>;
 }
 export interface TranscriptResponse {
     text: string;

package/dist/utils.d.ts

@@ -1,6 +1,3 @@
+import { FetchParams } from './types';
 export declare function retrieveVideoId(videoId: string): string;
-export declare function defaultFetch({ url, lang, userAgent, }: {
-    url: string;
-    lang?: string;
-    userAgent?: string;
-}): Promise<Response>;
+export declare function defaultFetch(params: FetchParams): Promise<Response>;

package/dist/youtube-transcript-plus.js

@@ -36,6 +36,7 @@ typeof SuppressedError === "function" ? SuppressedError : function (error, suppr
 const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36';
 const RE_YOUTUBE = /(?:v=|\/|v\/|embed\/|watch\?.*v=|youtu\.be\/|\/v\/|e\/|watch\?.*vi?=|\/embed\/|\/v\/|vi?\/|watch\?.*vi?=|youtu\.be\/|\/vi?\/|\/e\/)([a-zA-Z0-9_-]{11})/i;
 const RE_XML_TRANSCRIPT = /<text start="([^"]*)" dur="([^"]*)">([^<]*)<\/text>/g;
+const DEFAULT_CACHE_TTL = 3600000; // 1 hour in milliseconds
 
 class YoutubeTranscriptTooManyRequestError extends Error {
     constructor() {
@@ -84,16 +85,23 @@ function retrieveVideoId(videoId) {
     }
     throw new YoutubeTranscriptInvalidVideoIdError();
 }
-function defaultFetch(_a) {
-    return __awaiter(this, arguments, void 0, function* ({ url, lang, userAgent, }) {
-        return fetch(url, {
-            headers: Object.assign(Object.assign({}, (lang && { 'Accept-Language': lang })), { 'User-Agent': userAgent || DEFAULT_USER_AGENT }),
-        });
+function defaultFetch(params) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { url, lang, userAgent, method = 'GET', body, headers = {} } = params;
+        const fetchHeaders = Object.assign(Object.assign({ 'User-Agent': userAgent || DEFAULT_USER_AGENT }, (lang && { 'Accept-Language': lang })), headers);
+        const fetchOptions = {
+            method,
+            headers: fetchHeaders,
+        };
+        if (body && method === 'POST') {
+            fetchOptions.body = body;
+        }
+        return fetch(url, fetchOptions);
     });
 }
 
 class FsCache {
-    constructor(cacheDir = './cache', defaultTTL = 3600000) {
+    constructor(cacheDir = './cache', defaultTTL = DEFAULT_CACHE_TTL) {
         this.cacheDir = cacheDir;
         this.defaultTTL = defaultTTL;
         fs.mkdir(cacheDir, { recursive: true }).catch(() => { });
@@ -123,9 +131,8 @@ class FsCache {
 }
 
 class InMemoryCache {
-    constructor(defaultTTL = 3600000) {
+    constructor(defaultTTL = DEFAULT_CACHE_TTL) {
         this.cache = new Map();
-        // 1 hour default TTL
         this.defaultTTL = defaultTTL;
     }
     get(key) {
@@ -146,101 +153,154 @@ class InMemoryCache {
     }
 }
 
+/**
+ * Implementation notes:
+ * - Keeps the public surface identical.
+ * - Internals now use YouTube Innertube `player` to discover captionTracks instead of scraping the watch HTML.
+ * - Honors `lang`, custom fetch hooks (`videoFetch`, `transcriptFetch`), and optional cache strategy.
+ */
 class YoutubeTranscript {
     constructor(config) {
         this.config = config;
     }
     fetchTranscript(videoId) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p;
+            var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
             const identifier = retrieveVideoId(videoId);
-            const userAgent = ((_a = this.config) === null || _a === void 0 ? void 0 : _a.userAgent) || DEFAULT_USER_AGENT;
-            // Use custom fetch functions if provided, otherwise use defaultFetch
-            const videoFetch = ((_b = this.config) === null || _b === void 0 ? void 0 : _b.videoFetch) || defaultFetch;
-            const transcriptFetch = ((_c = this.config) === null || _c === void 0 ? void 0 : _c.transcriptFetch) || defaultFetch;
-            // Cache key based on video ID and language
-            const cacheKey = `transcript:${identifier}:${((_d = this.config) === null || _d === void 0 ? void 0 : _d.lang) || 'default'}`;
-            // Check cache first
-            if ((_e = this.config) === null || _e === void 0 ? void 0 : _e.cache) {
-                const cachedTranscript = yield this.config.cache.get(cacheKey);
-                if (cachedTranscript) {
-                    return JSON.parse(cachedTranscript);
+            const lang = (_a = this.config) === null || _a === void 0 ? void 0 : _a.lang;
+            const userAgent = (_c = (_b = this.config) === null || _b === void 0 ? void 0 : _b.userAgent) !== null && _c !== void 0 ? _c : DEFAULT_USER_AGENT;
+            // Cache lookup (if provided)
+            const cache = (_d = this.config) === null || _d === void 0 ? void 0 : _d.cache;
+            const cacheTTL = (_e = this.config) === null || _e === void 0 ? void 0 : _e.cacheTTL;
+            const cacheKey = `yt:transcript:${identifier}:${lang !== null && lang !== void 0 ? lang : ''}`;
+            if (cache) {
+                const cached = yield cache.get(cacheKey);
+                if (cached) {
+                    try {
+                        return JSON.parse(cached);
+                    }
+                    catch (_p) {
+                        // ignore parse errors and continue
+                    }
                 }
             }
+            // 1) Fetch the watch page to extract an Innertube API key (no interface change)
+            // Decide protocol once and reuse
             const protocol = ((_f = this.config) === null || _f === void 0 ? void 0 : _f.disableHttps) ? 'http' : 'https';
-            // Fetch the video page
-            const videoPageResponse = yield videoFetch({
-                url: `${protocol}://www.youtube.com/watch?v=${identifier}`,
-                lang: (_g = this.config) === null || _g === void 0 ? void 0 : _g.lang,
-                userAgent,
-            });
+            const watchUrl = `${protocol}://www.youtube.com/watch?v=${identifier}`;
+            const videoPageResponse = ((_g = this.config) === null || _g === void 0 ? void 0 : _g.videoFetch)
+                ? yield this.config.videoFetch({ url: watchUrl, lang, userAgent })
+                : yield defaultFetch({ url: watchUrl, lang, userAgent });
             if (!videoPageResponse.ok) {
                 throw new YoutubeTranscriptVideoUnavailableError(identifier);
             }
             const videoPageBody = yield videoPageResponse.text();
-            // Parse the video page to extract captions
-            const splittedHTML = videoPageBody.split('"captions":');
-            if (splittedHTML.length <= 1) {
-                if (videoPageBody.includes('class="g-recaptcha"')) {
-                    throw new YoutubeTranscriptTooManyRequestError();
-                }
-                if (!videoPageBody.includes('"playabilityStatus":')) {
-                    throw new YoutubeTranscriptVideoUnavailableError(identifier);
-                }
-                throw new YoutubeTranscriptDisabledError(identifier);
+            // Basic bot/recaptcha detection preserves old error behavior
+            if (videoPageBody.includes('class="g-recaptcha"')) {
+                throw new YoutubeTranscriptTooManyRequestError();
             }
-            const captions = (_h = (() => {
-                try {
-                    return JSON.parse(splittedHTML[1].split(',"videoDetails')[0].replace('\n', ''));
-                }
-                catch (e) {
-                    return undefined;
+            // 2) Extract Innertube API key from the page
+            const apiKeyMatch = videoPageBody.match(/"INNERTUBE_API_KEY":"([^"]+)"/) ||
+                videoPageBody.match(/INNERTUBE_API_KEY\\":\\"([^\\"]+)\\"/);
+            if (!apiKeyMatch) {
+                // If captions JSON wasn't present previously and we also can't find an API key,
+                // retain the disabled semantics for compatibility.
+                throw new YoutubeTranscriptNotAvailableError(identifier);
+            }
+            const apiKey = apiKeyMatch[1];
+            // 3) Call Innertube player as ANDROID client to retrieve captionTracks
+            const playerEndpoint = `https://www.youtube.com/youtubei/v1/player?key=${apiKey}`;
+            const playerBody = {
+                context: {
+                    client: {
+                        clientName: 'ANDROID',
+                        clientVersion: '20.10.38',
+                    },
+                },
+                videoId: identifier,
+            };
+            // Use configurable playerFetch for the POST to allow custom fetch logic.
+            const playerFetchParams = {
+                url: playerEndpoint,
+                method: 'POST',
+                lang,
+                userAgent,
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify(playerBody),
+            };
+            const playerRes = ((_h = this.config) === null || _h === void 0 ? void 0 : _h.playerFetch)
+                ? yield this.config.playerFetch(playerFetchParams)
+                : yield defaultFetch(playerFetchParams);
+            if (!playerRes.ok) {
+                throw new YoutubeTranscriptVideoUnavailableError(identifier);
+            }
+            const playerJson = yield playerRes.json();
+            const tracklist = (_k = (_j = playerJson === null || playerJson === void 0 ? void 0 : playerJson.captions) === null || _j === void 0 ? void 0 : _j.playerCaptionsTracklistRenderer) !== null && _k !== void 0 ? _k : playerJson === null || playerJson === void 0 ? void 0 : playerJson.playerCaptionsTracklistRenderer;
+            const tracks = tracklist === null || tracklist === void 0 ? void 0 : tracklist.captionTracks;
+            const isPlayableOk = ((_l = playerJson === null || playerJson === void 0 ? void 0 : playerJson.playabilityStatus) === null || _l === void 0 ? void 0 : _l.status) === 'OK';
+            // If `captions` is entirely missing, treat as "not available"
+            if (!(playerJson === null || playerJson === void 0 ? void 0 : playerJson.captions) || !tracklist) {
+                // If video is playable but captions aren’t provided, treat as "disabled"
+                if (isPlayableOk) {
+                    throw new YoutubeTranscriptDisabledError(identifier);
                 }
-            })()) === null || _h === void 0 ? void 0 : _h['playerCaptionsTracklistRenderer'];
-            if (!captions) {
+                // Otherwise we can’t assert they’re disabled; treat as "not available"
+                throw new YoutubeTranscriptNotAvailableError(identifier);
+            }
+            // If `captions` exists but there are zero tracks, treat as "disabled"
+            if (!Array.isArray(tracks) || tracks.length === 0) {
                 throw new YoutubeTranscriptDisabledError(identifier);
             }
-            if (!('captionTracks' in captions)) {
+            // Respect requested language or fallback to first track
+            const selectedTrack = lang ? tracks.find((t) => t.languageCode === lang) : tracks[0];
+            if (!selectedTrack) {
+                const available = tracks.map((t) => t.languageCode).filter(Boolean);
+                throw new YoutubeTranscriptNotAvailableLanguageError(lang, available, identifier);
+            }
+            // 4) Build transcript URL; prefer XML by stripping fmt if present
+            let transcriptURL = selectedTrack.baseUrl || selectedTrack.url;
+            if (!transcriptURL) {
                 throw new YoutubeTranscriptNotAvailableError(identifier);
             }
-            if (((_j = this.config) === null || _j === void 0 ? void 0 : _j.lang) &&
-                !captions.captionTracks.some((track) => { var _a; return track.languageCode === ((_a = this.config) === null || _a === void 0 ? void 0 : _a.lang); })) {
-                throw new YoutubeTranscriptNotAvailableLanguageError((_k = this.config) === null || _k === void 0 ? void 0 : _k.lang, captions.captionTracks.map((track) => track.languageCode), identifier);
+            transcriptURL = transcriptURL.replace(/&fmt=[^&]+$/, '');
+            if ((_m = this.config) === null || _m === void 0 ? void 0 : _m.disableHttps) {
+                transcriptURL = transcriptURL.replace(/^https:\/\//, 'http://');
             }
-            const captionURL = (((_l = this.config) === null || _l === void 0 ? void 0 : _l.lang)
-                ? captions.captionTracks.find((track) => { var _a; return track.languageCode === ((_a = this.config) === null || _a === void 0 ? void 0 : _a.lang); })
-                : captions.captionTracks[0]).baseUrl;
-            const transcriptURL = ((_m = this.config) === null || _m === void 0 ? void 0 : _m.disableHttps)
-                ? captionURL.replace('https://', 'http://')
-                : captionURL;
-            // Fetch the transcript
-            const transcriptResponse = yield transcriptFetch({
-                url: transcriptURL,
-                lang: (_o = this.config) === null || _o === void 0 ? void 0 : _o.lang,
-                userAgent,
-            });
+            // 5) Fetch transcript XML using the same hook surface as before
+            const transcriptResponse = ((_o = this.config) === null || _o === void 0 ? void 0 : _o.transcriptFetch)
+                ? yield this.config.transcriptFetch({ url: transcriptURL, lang, userAgent })
+                : yield defaultFetch({ url: transcriptURL, lang, userAgent });
             if (!transcriptResponse.ok) {
+                // Preserve legacy behavior
+                if (transcriptResponse.status === 429) {
+                    throw new YoutubeTranscriptTooManyRequestError();
+                }
                 throw new YoutubeTranscriptNotAvailableError(identifier);
             }
             const transcriptBody = yield transcriptResponse.text();
+            // 6) Parse XML into the existing TranscriptResponse shape
             const results = [...transcriptBody.matchAll(RE_XML_TRANSCRIPT)];
-            const transcript = results.map((result) => {
-                var _a, _b;
-                return ({
-                    text: result[3],
-                    duration: parseFloat(result[2]),
-                    offset: parseFloat(result[1]),
-                    lang: (_b = (_a = this.config) === null || _a === void 0 ? void 0 : _a.lang) !== null && _b !== void 0 ? _b : captions.captionTracks[0].languageCode,
-                });
-            });
-            // Store in cache if a strategy is provided
-            if ((_p = this.config) === null || _p === void 0 ? void 0 : _p.cache) {
-                yield this.config.cache.set(cacheKey, JSON.stringify(transcript), this.config.cacheTTL);
+            const transcript = results.map((m) => ({
+                text: m[3],
+                duration: parseFloat(m[2]),
+                offset: parseFloat(m[1]),
+                lang: lang !== null && lang !== void 0 ? lang : selectedTrack.languageCode,
+            }));
+            if (transcript.length === 0) {
+                throw new YoutubeTranscriptNotAvailableError(identifier);
+            }
+            // Cache store
+            if (cache) {
+                try {
+                    yield cache.set(cacheKey, JSON.stringify(transcript), cacheTTL);
+                }
+                catch (_q) {
+                    // non-fatal
+                }
             }
             return transcript;
         });
     }
-    // Add static method for new usage pattern
     static fetchTranscript(videoId, config) {
         return __awaiter(this, void 0, void 0, function* () {
             const instance = new YoutubeTranscript(config);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "youtube-transcript-plus",
-  "version": "1.0.4",
+  "version": "1.1.1",
   "description": "Fetch transcript from a YouTube video",
   "type": "module",
   "main": "dist/youtube-transcript-plus.js",
@@ -29,23 +29,26 @@
     ]
   },
   "devDependencies": {
-    "@types/jest": "^29.5.14",
+    "@types/jest": "^30.0.0",
     "https-proxy-agent": "^7.0.6",
     "husky": "^9.1.7",
-    "jest": "^29.7.0",
-    "lint-staged": "^15.5.0",
-    "prettier": "^3.5.3",
-    "rollup": "^4.37.0",
+    "jest": "^30.0.5",
+    "lint-staged": "^16.1.5",
+    "prettier": "^3.6.2",
+    "rollup": "^4.46.4",
     "rollup-plugin-typescript": "^1.0.1",
     "rollup-plugin-typescript2": "^0.36.0",
-    "ts-jest": "^29.3.0",
+    "ts-jest": "^29.4.1",
     "tslib": "^2.8.1",
-    "typescript": "^5.8.2"
+    "typescript": "^5.9.2"
   },
   "files": [
     "dist/*"
   ],
-  "repository": "https://github.com/ericmmartin/youtube-transcript-plus.git",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ericmmartin/youtube-transcript-plus.git"
+  },
   "publishConfig": {
     "access": "public"
   },