yt-grabber 1.5.0 → 1.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yt-grabber",
-  "version": "1.5.0",
+  "version": "1.6.0",
   "description": "Youtube Grabber",
   "main": "./dist/index.js",
   "repository": {

package/src/automations/Helpers.ts

@@ -1,5 +1,9 @@
+import fs from "fs-extra";
+import _isEmpty from "lodash/isEmpty";
 import {ElementHandle, Page} from "puppeteer";
 
+import {getProfilePath} from "../common/FileSystem";
+import {waitFor} from "../common/Helpers";
 import puppeteerOptions from "../common/PuppeteerOptions";
 
 export const navigateToPage = async (url: string, page: Page, timeout = puppeteerOptions.timeout) => {
@@ -13,3 +17,18 @@ export const clearInput = async (input: ElementHandle<Element>, page: Page) => {
     await input.click({clickCount: 3});
     await page.keyboard.press("Backspace");
 };
+
+export const setCookies = async (page: Page) => {
+    const cachedCookies = fs.readJSONSync(getProfilePath() + "/cookies.json", {throws: false});
+    
+    if (_isEmpty(cachedCookies)) {
+        await waitFor(3000);
+        const pageCookies = await page.cookies();
+    
+        fs.writeJSONSync(getProfilePath() + "/cookies.json", pageCookies, {spaces: 2});
+    
+        await page.setCookie(...pageCookies);
+    } else {
+        await page.setCookie(...cachedCookies);
+    }
+};

package/src/automations/Selectors.ts

@@ -1,11 +1,21 @@
+import _toLower from "lodash/toLower";
+
 export const AlbumsHrefSelector = "//ytmusic-app-layout//div[@id='content']/ytmusic-browse-response//ytmusic-section-list-renderer//div[@id='content-group']//div[contains(@class, 'header-renderer')]/yt-formatted-string/a[contains(text(), 'Album')]";
 
-export const AlbumFilterSelector = "//ytmusic-app-layout//div[@id='content']/ytmusic-browse-response//ytmusic-section-list-renderer/div[@id='header']//iron-selector//a[contains(@title, 'Album')]";
+export const SinglesHrefSelector = "//ytmusic-app-layout//div[@id='content']/ytmusic-browse-response//ytmusic-section-list-renderer//div[@id='content-group']//div[contains(@class, 'header-renderer')]/yt-formatted-string/a[contains(text(), 'Single')]";
+
+export const AlbumFilterSelector = "//ytmusic-app-layout//div[@id='content']/ytmusic-browse-response//ytmusic-section-list-renderer/div[@id='header']//iron-selector//ytmusic-chip-cloud-chip-renderer[not(@is-selected)]//a[contains(@title, 'Album')]";
+
+export const SingleFilterSelector = "//ytmusic-app-layout//div[@id='content']/ytmusic-browse-response//ytmusic-section-list-renderer/div[@id='header']//iron-selector//ytmusic-chip-cloud-chip-renderer[not(@is-selected)]//a[contains(@title, 'Single')]";
 
 export const AlbumLinkSelector = "//ytmusic-app-layout//div[@id='content']/ytmusic-browse-response//ytmusic-section-list-renderer/div[@id='contents']/ytmusic-grid-renderer/div[@id='items']/*[contains(@class, 'ytmusic-grid-renderer')]//a[contains(@class, 'yt-simple-endpoint') and contains(@class, 'yt-formatted-string')]";
 
+export const SingleLinkSelector = "//ytmusic-app-layout//div[@id='content']/ytmusic-browse-response//ytmusic-section-list-renderer/div[@id='contents']/ytmusic-grid-renderer/div[@id='items']/*[contains(@class, 'ytmusic-grid-renderer')]//a[contains(@class, 'yt-simple-endpoint') and contains(@class, 'yt-formatted-string')]";
+
 export const AlbumsDirectLinkSelector = "//ytmusic-app-layout//div[@id='content']/ytmusic-browse-response//ytmusic-section-list-renderer//div[@id='content-group']//div[contains(@class, 'header-renderer')]/yt-formatted-string[contains(text(), 'Album')]//ancestor::div[contains(@class, 'ytmusic-shelf')]//div[@id='items-wrapper']/ul/*[contains(@class, 'ytmusic-carousel')]//a[contains(@class, 'yt-simple-endpoint') and contains(@class, 'yt-formatted-string')]";
 
+export const SinglesDirectLinkSelector = "//ytmusic-app-layout//div[@id='content']/ytmusic-browse-response//ytmusic-section-list-renderer//div[@id='content-group']//div[contains(@class, 'header-renderer')]/yt-formatted-string/a[contains(text(), 'Single')]//ancestor::div[contains(@class, 'ytmusic-shelf')]//div[@id='items-wrapper']/ul/*[contains(@class, 'ytmusic-carousel')]//a[contains(@class, 'yt-simple-endpoint') and contains(@class, 'yt-formatted-string')]";
+
 export const YtMusicSearchInputSelector = "//ytmusic-app-layout//ytmusic-search-box//input[@id='input']";
 
 export const YtMusicArtistsChipSelector = "//ytmusic-app-layout//ytmusic-search-page//div[contains(@class, 'content')]//ytmusic-section-list-renderer//ytmusic-chip-cloud-renderer//iron-selector[@id='chips']//a[contains(@class, 'ytmusic-chip-cloud-chip-renderer')]//yt-formatted-string[contains(text(), 'Wykonawcy') or contains(text(), 'Artists')]//ancestor::a";
@@ -16,4 +26,14 @@ export const YtMusicAlbumsChipSelector = "//ytmusic-app-layout//ytmusic-search-p
 
 export const YtMusicSongsChipSelector = "//ytmusic-app-layout//ytmusic-search-page//div[contains(@class, 'content')]//ytmusic-section-list-renderer//ytmusic-chip-cloud-renderer//iron-selector[@id='chips']//a[contains(@class, 'ytmusic-chip-cloud-chip-renderer')]//yt-formatted-string[contains(text(), 'Utwory') or contains(text(), 'Songs')]//ancestor::a";
 
-export const YtMusicSearchResultsSelector = "//ytmusic-app-layout//div[@id='content']//ytmusic-search-page//div[@id='contents']//ytmusic-shelf-renderer//div[@id='contents']//ytmusic-responsive-list-item-renderer/a";
+export const YtMusicSearchResultsArtistsSelector = "//ytmusic-app-layout//div[@id='content']//ytmusic-search-page//div[@id='contents']//ytmusic-shelf-renderer//div[@id='contents']//ytmusic-responsive-list-item-renderer";
+
+export const YtMusicSearchResultsArtistsLinkSelector = "//ytmusic-app-layout//div[@id='content']//ytmusic-search-page//div[@id='contents']//ytmusic-shelf-renderer//div[@id='contents']//ytmusic-responsive-list-item-renderer/a";
+
+export const YtMusicArtistRelativeThumbnailSelector = ".//ytmusic-thumbnail-renderer//img";
+
+export const YtMusicArtistRelativeNameSelector = ".//div[contains(@class, 'title-column')]//yt-formatted-string";
+
+export const YtMusicArtistRelativeLinkSelector = ".//a";
+
+export const getYtMusicSearchResultsArtistsSelector = (artist: string) => `//ytmusic-app-layout//div[@id='content']//ytmusic-search-page//div[@id='contents']//ytmusic-shelf-renderer//div[@id='contents']//ytmusic-responsive-list-item-renderer//div[contains(@class, 'title-column')]/yt-formatted-string//text()[translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZÄÖÜÉÈÊÀÁÂÒÓÔÙÚÛÇÅÏÕÑŒĄĆĘŁŃÓŚŹŻ', 'abcdefghijklmnopqrstuvwxyzäöüéèêàáâòóôùúûçåïõñœąćęłńóśźż')='${_toLower(artist)}' or translate(., 'ABCDEFGHIJKLMNOPQRSTUVWXYZÄÖÜÉÈÊÀÁÂÒÓÔÙÚÛÇÅÏÕÑŒĄĆĘŁŃÓŚŹŻ', 'abcdefghijklmnopqrstuvwxyzäöüéèêàáâòóôùúûçåïõñœąćęłńóśźż')='the ${_toLower(artist)}']//ancestor::ytmusic-responsive-list-item-renderer`;

package/src/automations/Youtube.ts

@@ -1,21 +1,19 @@
-import fs from "fs-extra";
+import {i18n as i18next} from "i18next";
 import _forEach from "lodash/forEach";
 import _includes from "lodash/includes";
 import _isEmpty from "lodash/isEmpty";
 import _map from "lodash/map";
 import _merge from "lodash/merge";
 import _replace from "lodash/replace";
-import {Browser, Page, TimeoutError} from "puppeteer";
+import {Browser, LaunchOptions, Page, TimeoutError} from "puppeteer";
 import puppeteer from "puppeteer-extra";
 import StealthPlugin from "puppeteer-extra-plugin-stealth";
 
-import {getProfilePath} from "../common/FileSystem";
-import {waitFor} from "../common/Helpers";
-import {GetYoutubeResult} from "../common/Messaging";
-import puppeteerOptions from "../common/PuppeteerOptions";
-import {IReporter, Reporter} from "../common/Reporter";
+import {GetYoutubeParams, GetYoutubeResult} from "../common/Messaging";
+import puppeteerOptions, {UserAgent} from "../common/PuppeteerOptions";
+import {IReporter, ProgressInfo, Reporter} from "../common/Reporter";
 import {MessageHandlerParams} from "../messaging/MessageChannel";
-import {navigateToPage} from "./Helpers";
+import {navigateToPage, setCookies} from "./Helpers";
 import {
     AlbumFilterSelector, AlbumLinkSelector, AlbumsDirectLinkSelector, AlbumsHrefSelector
 } from "./Selectors";
@@ -33,79 +31,13 @@ export const execute = async (parameters: MessageHandlerParams) => {
     });
 
     try {
-        const result: GetYoutubeResult = {warnings: [], errors: [], values: []};
-        const userAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Mobile Safari/537.3";
+        
         await Promise.race([
-            (async () => {
-                await i18n.changeLanguage(params.lang);
-
-                reporter = new Reporter(onUpdate);
-                reporter.start(i18n.t("starting"));
-                browser = await puppeteer.launch(_merge(puppeteerOptions, options));
-                [page] = await browser.pages();
-
-                await page.setUserAgent(userAgent);
-
-                const cachedCookies = fs.readJSONSync(getProfilePath() + "/cookies.json", {throws: false});
-
-                if (_isEmpty(cachedCookies)) {
-                    await waitFor(3000);
-                    const pageCookies = await page.cookies();
-
-                    fs.writeJSONSync(getProfilePath() + "/cookies.json", pageCookies, {spaces: 2});
-
-                    await page.setCookie(...pageCookies);
-                } else {
-                    await page.setCookie(...cachedCookies);
-                }
-
-                await navigateToPage(params.url, page);
-
-                const process = async (urlToProcess: string) => {
-                    const results: string[] = [];
-
-                    try {
-                        await navigateToPage(urlToProcess, page);
-
-                        const element = await page.waitForSelector(`::-p-xpath(${AlbumsHrefSelector})`, {timeout: 1000});
-                        const albumsUrl = await element.evaluate((el) => el.getAttribute("href"));
-
-                        await navigateToPage(`${params.url}/${albumsUrl}`, page);
-                        const albumFilterButton = await page.waitForSelector(`::-p-xpath(${AlbumFilterSelector})`, {timeout: 1000});
-
-                        albumFilterButton.click();
-                        await page.waitForNetworkIdle();
-
-                        const items = await page.$$eval(`xpath/${AlbumLinkSelector}`, (elements) => elements.map((el) => el.getAttribute("href")));
-
-                        for (const item of items) {
-                            results.push(`${params.url}/${item}`);
-                        }
-
-                        return results;
-                    } catch (error) {
-                        const items = await page.$$eval(`xpath/${AlbumsDirectLinkSelector}`, (elements) => elements.map((el) => el.getAttribute("href")));
-
-                        for (const item of items) {
-                            results.push(`${params.url}/${item}`);
-                        }
-
-                        return results;
-                    }
-                };
-
-                for (const u of params.values) {
-                    const data = await process(u);
-
-                    result.values.push(...data);
-                }
-
-                reporter.finish("done", result);
-            })(),
+            run(params, options, i18n, onUpdate),
             abortPromise,
         ]);
     } catch (error: any) {
-        const result: GetYoutubeResult = {errors: []};
+        const result: GetYoutubeResult = {errors: [], sources: params.values};
         if (error.message === "aborted") {
             throw error;
         }
@@ -126,6 +58,62 @@ export const execute = async (parameters: MessageHandlerParams) => {
     }
 };
 
+const run = async (params: GetYoutubeParams, options: LaunchOptions, i18n: i18next, onUpdate: (data: ProgressInfo<GetYoutubeResult>) => void) => {
+    const result: GetYoutubeResult = {warnings: [], errors: [], values: [], sources: params.values};
+    
+    await i18n.changeLanguage(params.lang);
+
+    reporter = new Reporter(onUpdate);
+    reporter.start(i18n.t("starting"));
+    browser = await puppeteer.launch(_merge(puppeteerOptions, options));
+    [page] = await browser.pages();
+
+    await page.setUserAgent(UserAgent);
+    await setCookies(page);
+    await navigateToPage(params.url, page);
+
+    const process = async (urlToProcess: string) => {
+        const results: string[] = [];
+
+        try {
+            await navigateToPage(urlToProcess, page);
+
+            const element = await page.waitForSelector(`::-p-xpath(${AlbumsHrefSelector})`, {timeout: 1000});
+            const albumsUrl = await element.evaluate((el) => el.getAttribute("href"));
+
+            await navigateToPage(`${params.url}/${albumsUrl}`, page);
+            const albumFilterButton = await page.waitForSelector(`::-p-xpath(${AlbumFilterSelector})`, {timeout: 1000});
+
+            albumFilterButton.click();
+            await page.waitForNetworkIdle();
+
+            const items = await page.$$eval(`xpath/${AlbumLinkSelector}`, (elements) => elements.map((el) => el.getAttribute("href")));
+
+            for (const item of items) {
+                results.push(`${params.url}/${item}`);
+            }
+
+            return results;
+        } catch (error) {
+            const items = await page.$$eval(`xpath/${AlbumsDirectLinkSelector}`, (elements) => elements.map((el) => el.getAttribute("href")));
+
+            for (const item of items) {
+                results.push(`${params.url}/${item}`);
+            }
+
+            return results;
+        }
+    };
+
+    for (const u of params.values) {
+        const data = await process(u);
+
+        result.values.push(...data);
+    }
+
+    reporter.finish("done", result);
+};
+
 const closeResources = async () => {
     if (page) await page.close();
     if (browser) await browser.close();

package/src/automations/YoutubeAlbums.ts

@@ -1,23 +1,21 @@
-import fs from "fs-extra";
+import {i18n as i18next} from "i18next";
 import _forEach from "lodash/forEach";
 import _includes from "lodash/includes";
 import _isEmpty from "lodash/isEmpty";
 import _map from "lodash/map";
 import _merge from "lodash/merge";
 import _replace from "lodash/replace";
-import {Browser, Page, TimeoutError} from "puppeteer";
+import {Browser, LaunchOptions, Page, TimeoutError} from "puppeteer";
 import puppeteer from "puppeteer-extra";
 import StealthPlugin from "puppeteer-extra-plugin-stealth";
 
-import {getProfilePath} from "../common/FileSystem";
-import {waitFor} from "../common/Helpers";
-import {GetYoutubeResult} from "../common/Messaging";
-import puppeteerOptions from "../common/PuppeteerOptions";
-import {IReporter, Reporter} from "../common/Reporter";
+import {GetYoutubeParams, GetYoutubeResult} from "../common/Messaging";
+import puppeteerOptions, {UserAgent} from "../common/PuppeteerOptions";
+import {IReporter, ProgressInfo, Reporter} from "../common/Reporter";
 import {MessageHandlerParams} from "../messaging/MessageChannel";
-import {clearInput, navigateToPage} from "./Helpers";
+import {clearInput, navigateToPage, setCookies} from "./Helpers";
 import {
-    YtMusicAlbumsChipSelector, YtMusicSearchInputSelector, YtMusicSearchResultsSelector
+    YtMusicAlbumsChipSelector, YtMusicSearchInputSelector, YtMusicSearchResultsArtistsLinkSelector
 } from "./Selectors";
 
 let page: Page;
@@ -34,72 +32,11 @@ export const execute = async (parameters: MessageHandlerParams) => {
 
     try {
         await Promise.race([
-            (async () => {
-                const result: GetYoutubeResult = {warnings: [], errors: [], values: []};
-                const userAgent = "Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Mobile Safari/537.3";
-                
-                reporter = new Reporter(onUpdate);
-                reporter.start(i18n.t("starting"));
-                browser = await puppeteer.launch(_merge(puppeteerOptions, options));
-                [page] = await browser.pages();
-
-                await page.setUserAgent(userAgent);
-
-                const cachedCookies = fs.readJSONSync(getProfilePath() + "/cookies.json", {throws: false});
-
-                if (_isEmpty(cachedCookies)) {
-                    await waitFor(3000);
-                    const pageCookies = await page.cookies();
-
-                    fs.writeJSONSync(getProfilePath() + "/cookies.json", pageCookies, {spaces: 2});
-
-                    await page.setCookie(...pageCookies);
-                } else {
-                    await page.setCookie(...cachedCookies);
-                }
-
-                await navigateToPage(params.url, page);
-
-                const process = async (album: string) => {
-                    const results: string[] = [];
-
-                    try {
-                        const searchInput = await page.waitForSelector(`::-p-xpath(${YtMusicSearchInputSelector})`, {timeout: 1000});
-                        await clearInput(searchInput, page);
-                        await searchInput.type(album);
-                        page.keyboard.press("Enter");
-                        await page.waitForNetworkIdle();
-
-                        const albumsChip = await page.waitForSelector(`::-p-xpath(${YtMusicAlbumsChipSelector})`, {timeout: 1000});
-
-                        albumsChip.click();
-                        await page.waitForNetworkIdle();
-                        await page.waitForSelector(`::-p-xpath(${YtMusicSearchResultsSelector})`, {timeout: 1000});
-
-                        const albumsElements = await page.$$(`::-p-xpath(${YtMusicSearchResultsSelector})`);
-                        const albumEl = albumsElements[0];
-                        const albumUrl = await albumEl.evaluate((el) => el.getAttribute("href"));
-
-                        results.push(`${params.url}/${albumUrl}`);
-
-                        return results;
-                    } catch (error) {
-                        return results;
-                    }
-                };
-
-                for (const a of params.values) {
-                    const data = await process(a);
-
-                    result.values.push(...data);
-                }
-
-                reporter.finish("done", result);
-            })(),
+            run(params, options, i18n, onUpdate),
             abortPromise,
         ]);
     } catch (error: any) {
-        const result: GetYoutubeResult = {errors: []};
+        const result: GetYoutubeResult = {errors: [], sources: params.values};
         if (error.message === "aborted") {
             throw error;
         }
@@ -120,6 +57,55 @@ export const execute = async (parameters: MessageHandlerParams) => {
     }
 };
 
+const run = async (params: GetYoutubeParams, options: LaunchOptions, i18n: i18next, onUpdate: (data: ProgressInfo<GetYoutubeResult>) => void) => {
+    const result: GetYoutubeResult = {warnings: [], errors: [], values: [], sources: params.values};
+                
+    reporter = new Reporter(onUpdate);
+    reporter.start(i18n.t("starting"));
+    browser = await puppeteer.launch(_merge(puppeteerOptions, options));
+    [page] = await browser.pages();
+
+    await page.setUserAgent(UserAgent);
+    await setCookies(page);
+    await navigateToPage(params.url, page);
+
+    const process = async (album: string) => {
+        const results: string[] = [];
+
+        try {
+            const searchInput = await page.waitForSelector(`::-p-xpath(${YtMusicSearchInputSelector})`, {timeout: 1000});
+            await clearInput(searchInput, page);
+            await searchInput.type(album);
+            page.keyboard.press("Enter");
+            await page.waitForNetworkIdle();
+
+            const albumsChip = await page.waitForSelector(`::-p-xpath(${YtMusicAlbumsChipSelector})`, {timeout: 1000});
+
+            albumsChip.click();
+            await page.waitForNetworkIdle();
+            await page.waitForSelector(`::-p-xpath(${YtMusicSearchResultsArtistsLinkSelector})`, {timeout: 1000});
+
+            const albumsElements = await page.$$(`::-p-xpath(${YtMusicSearchResultsArtistsLinkSelector})`);
+            const albumEl = albumsElements[0];
+            const albumUrl = await albumEl.evaluate((el) => el.getAttribute("href"));
+
+            results.push(`${params.url}/${albumUrl}`);
+
+            return results;
+        } catch (error) {
+            return results;
+        }
+    };
+
+    for (const a of params.values) {
+        const data = await process(a);
+
+        result.values.push(...data);
+    }
+
+    reporter.finish("done", result);
+};
+
 const closeResources = async () => {
     if (page) await page.close();
     if (browser) await browser.close();