shirayuki-anime-scraper-api 1.0.0

package/scrapeanime/homepage/latest/latest.js

@@ -0,0 +1,118 @@
+export default function scrapeLatest($, resolveUrl, source) {
+  const items = [];
+
+  const selectors = ['.flw-item', '.film-list .item', '.block_area-content .flw-item', '.film_list-wrap .flw-item'];
+
+  for (const sel of selectors) {
+    const found = $(sel);
+    if (!found || !found.length) continue;
+
+    found.slice(0, 15).each((i, el) => {
+      const el$ = $(el);
+      
+      let title = el$.find('.film-name a').attr('title') || 
+                 el$.find('.film-name a').text() || 
+                 el$.find('.dynamic-name').attr('data-jname') ||
+                 el$.find('h3 a').text() ||
+                 el$.find('.film-detail .film-name a').text() ||
+                 el$.find('a').attr('title') || null;
+      if (title) title = title.trim();
+
+      let href = el$.find('.film-name a').attr('href') || 
+                el$.find('.film-detail .film-name a').attr('href') ||
+                el$.find('a').first().attr('href') || '';
+      href = href ? resolveUrl(href) : null;
+
+      let image = null;
+      const imgEl = el$.find('.film-poster img, img').first();
+      if (imgEl && imgEl.length) {
+        image = imgEl.attr('data-src') || 
+               imgEl.attr('data-lazy') || 
+               imgEl.attr('src') || 
+               imgEl.attr('data-original') || null;
+      }
+      if (image) image = resolveUrl(image);
+
+      let subtitles = null;
+      const subEl = el$.find('.tick-item.tick-sub').first();
+      if (subEl.length) {
+        const subText = subEl.text().trim();
+        const subMatch = subText.match(/(\d+)/);
+        subtitles = subMatch ? subMatch[1] : null;
+      }
+
+      let dubbed = null;
+      const dubEl = el$.find('.tick-item.tick-dub').first();
+      if (dubEl.length) {
+        const dubText = dubEl.text().trim();
+        const dubMatch = dubText.match(/(\d+)/);
+        dubbed = dubMatch ? dubMatch[1] : null;
+      }
+
+      let episodes = null;
+      
+      let episodeEl = el$.find('.fdb-type').first();
+      
+      if (!episodeEl.length) {
+        episodeEl = el$.find('.fd-bar .fdb-type').first();
+      }
+      if (!episodeEl.length) {
+        episodeEl = el$.find('.film-detail .fd-bar .fdb-type').first();
+      }
+      if (!episodeEl.length) {
+        episodeEl = el$.find('*').filter((i, elem) => {
+          const text = $(elem).text().trim().toLowerCase();
+          return text.match(/ep\s*\d+/i) && $(elem).children().length === 0;
+        }).first();
+      }
+      
+      if (episodeEl.length) {
+        const episodeText = episodeEl.text().trim();
+        const epMatch = episodeText.match(/EP\s*(\d+)/i);
+        
+        if (epMatch) {
+          episodes = epMatch[1];
+        } else if (episodeText.toLowerCase().includes('movie')) {
+          episodes = "Movie";
+        } else if (episodeText.toLowerCase().includes('ova')) {
+          episodes = "OVA";
+        } else if (episodeText.toLowerCase().includes('special')) {
+          episodes = "Special";
+        } else {
+          const numMatch = episodeText.match(/(\d+)/);
+          if (numMatch) {
+            episodes = numMatch[1];
+          }
+        }
+      }
+      
+      if (!episodes && (subtitles || dubbed)) {
+        if (subtitles && dubbed && subtitles === dubbed) {
+          episodes = subtitles;
+        } else if (subtitles) {
+          episodes = subtitles;
+        } else if (dubbed) {
+          episodes = dubbed;
+        }
+      }
+
+      if (title || href) {
+        const item = {
+          title: title || null,
+          href: href || null,
+          image: image || null,
+          subtitles: subtitles || null,
+          dubbed: dubbed || null,
+          episodes: episodes || null,
+          source,
+          section: 'latest'
+        };
+        items.push(item);
+      }
+    });
+
+    if (items.length) break;
+  }
+
+  return items;
+}

package/scrapeanime/homepage/most_favorite/mostFavorite.js

@@ -0,0 +1,55 @@
+export default function scrapeMostFavorite($, resolveUrl, source) {
+  const items = [];
+
+  $('div.anif-block').each((i, block) => {
+    const block$ = $(block);
+    const header = block$.find('.anif-block-header').text() || '';
+    if (!/most\s*favorite/i.test(header)) return;
+
+    block$.find('.anif-block-ul ul.ulclear > li').slice(0, 6).each((j, li) => {
+      const el$ = $(li);
+      const a = el$.find('h3.film-name a').first();
+      let href = a.attr('href') || el$.find('a').first().attr('href') || '';
+      href = href ? resolveUrl(href) : null;
+
+      let title = a.attr('title') || a.attr('data-jname') || a.text() || null;
+      if (title) title = title.trim();
+
+      let img = null;
+      const poster = el$.find('.film-poster').first();
+      if (poster && poster.length) {
+        const imgEl = poster.find('img').first();
+        if (imgEl && imgEl.length) {
+          img = imgEl.attr('data-src') || imgEl.attr('data-lazy') || imgEl.attr('src') || imgEl.attr('data-original') || null;
+        }
+        if (!img) {
+          const style = poster.attr('style') || poster.find('a').attr('style') || '';
+          const m = /url\(['"]?(.*?)['"]?\)/.exec(style);
+          if (m && m[1]) img = m[1];
+        }
+      }
+      if (img) img = resolveUrl(img);
+
+      const dubText = el$.find('.tick .tick-item.tick-dub').text() || el$.find('.tick-item.tick-dub').text() || '';
+      const subText = el$.find('.tick .tick-item.tick-sub').text() || el$.find('.tick-item.tick-sub').text() || '';
+      const dub = (dubText || '').toString().replace(/[,\s"']/g, '').match(/(\d+)/);
+      const sub = (subText || '').toString().replace(/[,\s"']/g, '').match(/(\d+)/);
+
+      const fdi = el$.find('.fdi-item').text() || el$.find('.fd-infor .fdi-item').text() || '';
+      const tv = /\bTV\b/i.test(fdi);
+
+      items.push({
+        title: title || null,
+        href: href || null,
+        image: img || null,
+        dub: dub ? parseInt(dub[1], 10) : null,
+        sub: sub ? parseInt(sub[1], 10) : null,
+        tv: !!tv,
+        source,
+        section: 'most_favorite',
+      });
+    });
+  });
+
+  return items;
+}

package/scrapeanime/homepage/most_popular/mostPopular.js

@@ -0,0 +1,55 @@
+export default function scrapeMostPopular($, resolveUrl, source) {
+  const items = [];
+
+  $('div.anif-block').each((i, block) => {
+    const block$ = $(block);
+    const header = block$.find('.anif-block-header').text() || '';
+    if (!/most\s*popular/i.test(header)) return;
+
+    block$.find('.anif-block-ul ul.ulclear > li').slice(0, 6).each((j, li) => {
+      const el$ = $(li);
+      const a = el$.find('h3.film-name a').first();
+      let href = a.attr('href') || el$.find('a').first().attr('href') || '';
+      href = href ? resolveUrl(href) : null;
+
+      let title = a.attr('title') || a.attr('data-jname') || a.text() || null;
+      if (title) title = title.trim();
+
+      let img = null;
+      const poster = el$.find('.film-poster').first();
+      if (poster && poster.length) {
+        const imgEl = poster.find('img').first();
+        if (imgEl && imgEl.length) {
+          img = imgEl.attr('data-src') || imgEl.attr('data-lazy') || imgEl.attr('src') || imgEl.attr('data-original') || null;
+        }
+        if (!img) {
+          const style = poster.attr('style') || poster.find('a').attr('style') || '';
+          const m = /url\(['"]?(.*?)['"]?\)/.exec(style);
+          if (m && m[1]) img = m[1];
+        }
+      }
+      if (img) img = resolveUrl(img);
+
+      const dubText = el$.find('.tick .tick-item.tick-dub').text() || el$.find('.tick-item.tick-dub').text() || '';
+      const subText = el$.find('.tick .tick-item.tick-sub').text() || el$.find('.tick-item.tick-sub').text() || '';
+      const dub = (dubText || '').toString().replace(/[,\s"']/g, '').match(/(\d+)/);
+      const sub = (subText || '').toString().replace(/[,\s"']/g, '').match(/(\d+)/);
+
+      const fdi = el$.find('.fdi-item').text() || el$.find('.fd-infor .fdi-item').text() || '';
+      const tv = /\bTV\b/i.test(fdi);
+
+      items.push({
+        title: title || null,
+        href: href || null,
+        image: img || null,
+        dub: dub ? parseInt(dub[1], 10) : null,
+        sub: sub ? parseInt(sub[1], 10) : null,
+        tv: !!tv,
+        source,
+        section: 'most_popular',
+      });
+    });
+  });
+
+  return items;
+}

package/scrapeanime/homepage/recently_updated/recentlyUpdated.js

@@ -0,0 +1,56 @@
+export default function scrapeRecentlyUpdated($, resolveUrl, source) {
+  const items = [];
+
+  $('div.widget').each((i, widget) => {
+    const w$ = $(widget);
+    const title = w$.find('.widget-title .title, .widget-title h1.title').text() || w$.find('.widget-title').text() || '';
+    if (!/recently\s*updated/i.test(title)) return;
+
+    w$.find('.film-list .item').slice(0, 15).each((j, item) => {
+      const el$ = $(item);
+      const posterA = el$.find('a.poster').first();
+      const nameA = el$.find('a.name').first();
+
+      let href = posterA.attr('href') || nameA.attr('href') || '';
+      href = href ? resolveUrl(href) : null;
+
+      let titleText = nameA.attr('data-title') || nameA.attr('data-jtitle') || nameA.text() || posterA.attr('data-title') || null;
+      if (titleText) titleText = titleText.trim();
+
+      let img = null;
+      const imgEl = posterA.find('img').first();
+      if (imgEl && imgEl.length) img = imgEl.attr('data-src') || imgEl.attr('src') || imgEl.attr('data-lazy') || null;
+      if (!img) {
+        const style = posterA.attr('style') || posterA.find('div').attr('style') || '';
+        const m = /url\(['"]?(.*?)['"]?\)/.exec(style);
+        if (m && m[1]) img = m[1];
+      }
+      if (img) img = resolveUrl(img);
+
+      let episode = null;
+      let audio = null;
+      const status = el$.find('.status').first();
+      if (status && status.length) {
+        const epText = status.find('.ep').text() || status.find('.epi').text() || '';
+        const epMatch = (epText || '').toString().match(/(\d+)/);
+        if (epMatch) episode = parseInt(epMatch[1], 10);
+
+        const subEl = status.find('.sub').first();
+        const dubEl = status.find('.dub').first();
+        if (subEl && subEl.length) audio = 'sub';
+        else if (dubEl && dubEl.length) audio = 'dub';
+        else {
+          const sText = status.text() || '';
+          if (/\bSUB\b/i.test(sText)) audio = 'sub';
+          else if (/\bDUB\b/i.test(sText)) audio = 'dub';
+        }
+      }
+
+      if (href || titleText) {
+        items.push({ title: titleText || null, href: href || null, image: img || null, episode: episode, source, section: 'recently_updated' });
+      }
+    });
+  });
+
+  return items;
+}

package/scrapeanime/homepage/scrapeAnimeDetails.js

@@ -0,0 +1,128 @@
+import { fetchAndLoad } from '../../service/scraperService.js';
+
+export async function scrapeAnimeDetails(animeUrl) {
+  if (!animeUrl) return null;
+
+  try {
+    const $ = await fetchAndLoad(animeUrl);
+    
+    const details = {
+      description: null,
+      synonyms: null,
+      aired: null,
+      premiered: null,
+      duration: null,
+      status: null,
+      malScore: null,
+      genres: [],
+      studios: [],
+      producers: []
+    };
+
+    $('.anisc-info .item').each((i, item) => {
+      const $item = $(item);
+      const label = $item.find('.item-head').text().trim().toLowerCase();
+      const content = $item.find('.name, .text').text().trim();
+
+      switch (label) {
+        case 'synonyms:':
+        case 'japanese:':
+          if (content && content !== 'N/A') {
+            details.synonyms = content;
+          }
+          break;
+        case 'aired:':
+          if (content && content !== 'N/A') {
+            details.aired = content;
+          }
+          break;
+        case 'premiered:':
+          if (content && content !== 'N/A') {
+            details.premiered = content;
+          }
+          break;
+        case 'duration:':
+          if (content && content !== 'N/A') {
+            details.duration = content;
+          }
+          break;
+        case 'status:':
+          if (content && content !== 'N/A') {
+            details.status = content;
+          }
+          break;
+        case 'mal score:':
+        case 'score:':
+          if (content && content !== 'N/A' && !isNaN(parseFloat(content))) {
+            details.malScore = parseFloat(content);
+          }
+          break;
+        case 'genres:':
+          $item.find('a').each((j, genreLink) => {
+            const genre = $(genreLink).text().trim();
+            if (genre) details.genres.push(genre);
+          });
+          break;
+        case 'studios:':
+          $item.find('a').each((j, studioLink) => {
+            const studio = $(studioLink).text().trim();
+            if (studio) details.studios.push(studio);
+          });
+          break;
+        case 'producers:':
+          $item.find('a').each((j, producerLink) => {
+            const producer = $(producerLink).text().trim();
+            if (producer) details.producers.push(producer);
+          });
+          break;
+      }
+    });
+
+    const descriptionSelectors = [
+      '.film-description .text',
+      '.anisc-detail .film-description .text',
+      '.anime-synopsis .text',
+      '.description .text',
+      '.overview .text',
+      '[data-content="description"]',
+      '.anisc-info .item .text'
+    ];
+
+    for (const selector of descriptionSelectors) {
+      const descElement = $(selector);
+      if (descElement.length && descElement.text().trim()) {
+        let desc = descElement.text().trim();
+        // Clean up common unwanted text
+        desc = desc.replace(/^(Description|Synopsis|Overview):\s*/i, '');
+        if (desc && desc.length > 50) { 
+          details.description = desc;
+          break;
+        }
+      }
+    }
+
+    if (!details.malScore) {
+      const scoreText = $('.film-stats .tick .tick-pg, .score').text();
+      const scoreMatch = scoreText.match(/(\d+\.?\d*)/);
+      if (scoreMatch) {
+        details.malScore = parseFloat(scoreMatch[1]);
+      }
+    }
+
+    if (details.genres.length === 0) {
+      $('.film-stats .item .name a, .genres a').each((i, genreEl) => {
+        const genre = $(genreEl).text().trim();
+        if (genre && !details.genres.includes(genre)) {
+          details.genres.push(genre);
+        }
+      });
+    }
+
+    return details;
+  } catch (error) {
+    console.error(`Error scraping anime details from ${animeUrl}:`, error.message);
+    return null;
+  }
+}
+
+export default { scrapeAnimeDetails };

package/scrapeanime/homepage/scrapehomepage.js

@@ -0,0 +1,2 @@
+export { default } from './scrapeservice.js';
+export { scrapeHomepage } from './scrapeservice.js';

package/scrapeanime/homepage/scrapeservice.js

@@ -0,0 +1,158 @@
+import scrapeTopAiring from './top_airing/topAiring.js';
+import scrapeMostPopular from './most_popular/mostPopular.js';
+import scrapeMostFavorite from './most_favorite/mostFavorite.js';
+import scrapeRecentlyUpdated from './recently_updated/recentlyUpdated.js';
+import scrapeSlider from './slider/slider.js';
+import scrapeLatest from './latest/latest.js';
+import scrapeTrending from './trending/trending.js';
+import { fetchAndLoad, resolveUrlFactory } from '../../service/scraperService.js';
+
+const cache = new Map();
+const CACHE_TTL = 2 * 60 * 1000; 
+
+function getCacheKey(url, includeDetails) {
+  return `${url}_${includeDetails}`;
+}
+
+function getFromCache(key) {
+  const cached = cache.get(key);
+  if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
+    return cached.data;
+  }
+  return null;
+}
+
+function setCache(key, data) {
+  cache.set(key, {
+    data,
+    timestamp: Date.now()
+  });
+  
+  if (cache.size > 10) {
+    const oldestKeys = Array.from(cache.keys()).slice(0, 5);
+    oldestKeys.forEach(key => cache.delete(key));
+  }
+}
+
+async function scrapeSite(url, base, source, includeDetails = false) {
+	const cacheKey = getCacheKey(url, includeDetails);
+	const cached = getFromCache(cacheKey);
+	
+	if (cached) {
+		return cached;
+	}
+
+	const $ = await fetchAndLoad(url);
+	const resolveUrl = resolveUrlFactory(base);
+
+	const items = [];
+
+	try {
+		const top = await scrapeTopAiring($, resolveUrl, source, includeDetails);
+		if (top && top.length) items.push(...top);
+	} catch (e) {
+	}
+
+	try {
+		const popular = await scrapeMostPopular($, resolveUrl, source, includeDetails);
+		if (popular && popular.length) items.push(...popular);
+	} catch (e) {
+	}
+
+	try {
+		const fav = await scrapeMostFavorite($, resolveUrl, source, includeDetails);
+		if (fav && fav.length) items.push(...fav);
+	} catch (e) {
+	}
+
+	try {
+		if (source === '123animehub') {
+			const recent = scrapeRecentlyUpdated($, resolveUrl, source);
+			if (recent && recent.length) items.push(...recent);
+		}
+	} catch (e) {
+	}
+
+	try {
+		if (source !== '123animehub') {
+			const slider = scrapeSlider($, resolveUrl, source);
+			if (slider && slider.length) items.push(...slider);
+		}
+	} catch (e) {
+	}
+
+	try {
+		const latest = scrapeLatest($, resolveUrl, source);
+		if (latest && latest.length) items.push(...latest);
+	} catch (e) {
+	}
+
+	try {
+		const trending = await scrapeTrending($, resolveUrl, source, includeDetails);
+		if (trending && trending.length) items.push(...trending);
+	} catch (e) {
+	}
+
+	const seen = new Set();
+	const deduped = [];
+	for (const it of items) {
+		const key = (it.href || it.title || it.image || '').toString().toLowerCase();
+		if (!key) continue;
+		if (!seen.has(key)) {
+			seen.add(key);
+			deduped.push(it);
+		}
+	}
+
+	setCache(cacheKey, deduped);
+	
+	return deduped;
+}
+
+export async function scrapeHomepage(includeDetails = false) {
+	const tasks = [
+		scrapeSite('https://hianime.to/home', 'https://hianime.to', 'hianime', includeDetails),
+		scrapeSite('https://123animehub.cc/home', 'https://123animehub.cc', '123animehub', includeDetails),
+	];
+
+	const results = await Promise.allSettled(tasks);
+
+	const combined = [];
+	const errors = [];
+
+	if (results[0].status === 'fulfilled') combined.push(...results[0].value);
+	else errors.push({ source: 'hianime', error: String(results[0].reason) });
+
+	if (results[1].status === 'fulfilled') combined.push(...results[1].value);
+	else errors.push({ source: '123animehub', error: String(results[1].reason) });
+
+	const seen = new Set();
+	const deduped = [];
+	for (const it of combined) {
+		const key = (it.href || it.title || it.image || '').toString().toLowerCase();
+		if (!key) continue;
+		if (!seen.has(key)) {
+			seen.add(key);
+			deduped.push(it);
+		}
+	}
+
+	const sectionTotals = {};
+	for (const it of deduped) {
+		const sec = it.section || 'unknown';
+		sectionTotals[sec] = (sectionTotals[sec] || 0) + 1;
+	}
+
+	const sectionCounters = {};
+	const indexed = deduped.map((item) => {
+		const sec = item.section || 'unknown';
+		sectionCounters[sec] = (sectionCounters[sec] || 0) + 1;
+		return { index: sectionCounters[sec], ...item };
+	});
+
+	const result = { success: true, data: indexed, total: indexed.length, sectionTotals };
+	if (errors.length) result.errors = errors;
+	return result;
+}
+
+export default scrapeHomepage;