shirayuki-anime-scraper-api 1.0.0

package/scrapeanime/homepage/slider/slider.js

@@ -0,0 +1,151 @@
+export default function scrapeSlider($, resolveUrl, source) {
+  const items = [];
+
+  const selectors = ['.swiper-slide', '.slider .item', '.home-slider .slide', '.featured-slider .item', '.swiper-slide.item-qtip', '.film-poster'];
+
+  for (const sel of selectors) {
+    const found = $(sel);
+    if (!found || !found.length) continue;
+
+    found.each((i, el) => {
+      const el$ = $(el);
+      const a = el$.find('a').first();
+      let href = a.attr('href') || el$.attr('href') || '';
+      href = href ? resolveUrl(href) : null;
+
+      let title = el$.find('.desi-head-title').text() ||
+        el$.find('.film-title').text() ||
+        el$.find('.title').text() ||
+        el$.find('h3').text() ||
+        a.attr('title') ||
+        el$.find('img').attr('alt') ||
+        el$.find('.film-title').attr('data-iname') || null;
+      if (title) title = title.trim();
+
+      let img = null;
+      const imgEl = el$.find('img').first();
+      if (imgEl && imgEl.length) img = imgEl.attr('data-src') || imgEl.attr('data-lazy') || imgEl.attr('src') || imgEl.attr('data-original') || null;
+      if (!img) img = el$.attr('data-background') || el$.attr('data-image') || null;
+      if (img) img = resolveUrl(img);
+
+      let description = el$.find('.desi-description').text() ||
+        el$.closest('.swiper-slide').find('.desi-description').text() ||
+        el$.parent().find('.desi-description').text() ||
+        el$.find('.description').text() ||
+        el$.find('.synopsis').text() ||
+        el$.find('.summary').text() ||
+        el$.find('[class*="desc"]').text() || null;
+      if (description) description = description.trim();
+
+      let isTV = el$.find('.scd-item').filter((i, elem) => {
+        const $elem = $(elem);
+        return $elem.find('.fas.fa-play-circle').length > 0 && $elem.text().includes('TV');
+      }).length > 0 ||
+        el$.closest('.swiper-slide').find('.scd-item').filter((i, elem) => {
+          const $elem = $(elem);
+          return $elem.find('.fas.fa-play-circle').length > 0 && $elem.text().includes('TV');
+        }).length > 0 ||
+        el$.find('[class*="tv"]').length > 0 ||
+        el$.find('.film-detail .fd-infor .fdi-item').filter((i, elem) => {
+          return $(elem).text().toLowerCase().includes('tv');
+        }).length > 0 ||
+        (title && title.toLowerCase().includes('season'));
+
+      let duration = null;
+      let durationEl = el$.find('.scd-item').filter((i, elem) => {
+        return $(elem).find('.fas.fa-clock').length > 0;
+      });
+      if (!durationEl.length) {
+        durationEl = el$.closest('.swiper-slide').find('.scd-item').filter((i, elem) => {
+          return $(elem).find('.fas.fa-clock').length > 0;
+        });
+      }
+      if (durationEl.length) {
+        const durationText = durationEl.text().trim();
+        const match = durationText.match(/(\d+)m/i);
+        duration = match ? match[1] : null;
+      }
+
+      let releaseDate = null;
+      let dateEl = el$.find('.scd-item').filter((i, elem) => {
+        return $(elem).find('.fas.fa-calendar').length > 0;
+      });
+      if (!dateEl.length) {
+        dateEl = el$.closest('.swiper-slide').find('.scd-item').filter((i, elem) => {
+          return $(elem).find('.fas.fa-calendar').length > 0;
+        });
+      }
+      if (dateEl.length) {
+        const dateText = dateEl.text().trim();
+        releaseDate = dateText || null;
+      }
+
+      let quality = el$.find('.scd-item .quality').text() ||
+        el$.closest('.swiper-slide').find('.scd-item .quality').text() ||
+        el$.find('.quality').text() ||
+        el$.find('[class*="quality"]').text() ||
+        el$.find('.film-poster-quality').text() ||
+        el$.find('.badge').text() ||
+        el$.find('.resolution').text() || null;
+      if (quality) quality = quality.trim();
+
+      let subtitles = null;
+      let dubbed = false;
+
+      let subEl = el$.find('.tick-item').filter((i, elem) => {
+        return $(elem).find('.fas.fa-closed-captioning').length > 0;
+      });
+      if (!subEl.length) {
+        subEl = el$.closest('.swiper-slide').find('.tick-item').filter((i, elem) => {
+          return $(elem).find('.fas.fa-closed-captioning').length > 0;
+        });
+      }
+      if (subEl.length) {
+        const subText = subEl.text().trim();
+        const subMatch = subText.match(/(\d+)/);
+        subtitles = subMatch ? subMatch[1] : null;
+      }
+
+      let dubEl = el$.find('.tick-item').filter((i, elem) => {
+        const $elem = $(elem);
+        const text = $elem.text().toLowerCase();
+        return text.includes('dub') || text.includes('english') || $elem.hasClass('dub');
+      });
+      if (!dubEl.length) {
+        dubEl = el$.closest('.swiper-slide').find('.tick-item').filter((i, elem) => {
+          const $elem = $(elem);
+          const text = $elem.text().toLowerCase();
+          return text.includes('dub') || text.includes('english') || $elem.hasClass('dub');
+        });
+      }
+      if (dubEl.length) {
+        dubbed = true;
+      }
+
+      if (href || title) {
+        // Only include items from hianime source or with hianime URLs and limit to first 8 items
+        if (href && (href.includes('hianime.to') || href.includes('/watch/')) && items.length < 8) {
+          const item = {
+            title: title || null,
+            href: href || null,
+            image: img || null,
+            description: description || null,
+            isTV: isTV || false,
+            duration: duration || null,
+            releaseDate: releaseDate || null,
+            quality: quality || null,
+            subtitles: subtitles || null,
+            dubbed: dubbed || false,
+            source,
+            section: 'slider'
+          };
+          items.push(item);
+        }
+      }
+    });
+
+    if (items.length) break;
+  }
+
+  return items;
+}

package/scrapeanime/homepage/top_airing/topAiring.js

@@ -0,0 +1,55 @@
+export default function scrapeTopAiring($, resolveUrl, source) {
+  const items = [];
+
+  $('div.anif-block').each((i, block) => {
+    const block$ = $(block);
+    const header = block$.find('.anif-block-header').text() || '';
+    if (!/top\s*airing/i.test(header)) return;
+
+    block$.find('.anif-block-ul ul.ulclear > li').slice(0, 6).each((j, li) => {
+      const el$ = $(li);
+      const a = el$.find('h3.film-name a').first();
+      let href = a.attr('href') || el$.find('a').first().attr('href') || '';
+      href = href ? resolveUrl(href) : null;
+
+      let title = a.attr('title') || a.attr('data-jname') || a.text() || null;
+      if (title) title = title.trim();
+
+      let img = null;
+      const poster = el$.find('.film-poster').first();
+      if (poster && poster.length) {
+        const imgEl = poster.find('img').first();
+        if (imgEl && imgEl.length) {
+          img = imgEl.attr('data-src') || imgEl.attr('data-lazy') || imgEl.attr('src') || imgEl.attr('data-original') || null;
+        }
+        if (!img) {
+          const style = poster.attr('style') || poster.find('a').attr('style') || '';
+          const m = /url\(['"]?(.*?)['"]?\)/.exec(style);
+          if (m && m[1]) img = m[1];
+        }
+      }
+      if (img) img = resolveUrl(img);
+
+      const dubText = el$.find('.tick .tick-item.tick-dub').text() || el$.find('.tick-item.tick-dub').text() || '';
+      const subText = el$.find('.tick .tick-item.tick-sub').text() || el$.find('.tick-item.tick-sub').text() || '';
+      const dub = (dubText || '').toString().replace(/[,\s"']/g, '').match(/(\d+)/);
+      const sub = (subText || '').toString().replace(/[,\s"']/g, '').match(/(\d+)/);
+
+      const fdi = el$.find('.fdi-item').text() || el$.find('.fd-infor .fdi-item').text() || '';
+      const tv = /\bTV\b/i.test(fdi);
+
+      items.push({
+        title: title || null,
+        href: href || null,
+        image: img || null,
+        dub: dub ? parseInt(dub[1], 10) : null,
+        sub: sub ? parseInt(sub[1], 10) : null,
+        tv: !!tv,
+        source,
+        section: 'top_airing',
+      });
+    });
+  });
+
+  return items;
+}

package/scrapeanime/homepage/trending/trending.js

@@ -0,0 +1,59 @@
+export default function scrapeTrending($, resolveUrl, source) {
+  const items = [];
+
+  const selectors = ['.swiper-slide.item-qtip', '.trending-list .swiper-slide', '.block_area-content .swiper-slide'];
+
+  for (const sel of selectors) {
+    const found = $(sel);
+    if (!found || !found.length) continue;
+
+    found.slice(0, 6).each((i, el) => {
+      const el$ = $(el);
+      
+      let title = el$.find('.film-title.dynamic-name').attr('data-jname') || 
+                 el$.find('.film-title').text() || 
+                 el$.find('[data-jname]').attr('data-jname') ||
+                 el$.find('a').attr('title') || 
+                 el$.find('.title').text() || null;
+      if (title) title = title.trim();
+
+      let href = el$.find('a.film-poster').attr('href') || 
+                el$.find('a').first().attr('href') || '';
+      href = href ? resolveUrl(href) : null;
+
+      let image = null;
+      const imgEl = el$.find('.film-poster-img').first();
+      if (imgEl && imgEl.length) {
+        image = imgEl.attr('data-src') || 
+               imgEl.attr('data-lazy') || 
+               imgEl.attr('src') || 
+               imgEl.attr('data-original') || null;
+      }
+      if (image) image = resolveUrl(image);
+
+      let number = null;
+      const numberEl = el$.find('.number').first();
+      if (numberEl.length) {
+        const numberText = numberEl.text().trim();
+        const numberMatch = numberText.match(/(\d+)/);
+        number = numberMatch ? parseInt(numberMatch[1]) : null;
+      }
+
+      if (title && href) {
+        const item = {
+          title: title || null,
+          href: href || null,
+          image: image || null,
+          number: number || null,
+          source,
+          section: 'trending'
+        };
+        items.push(item);
+      }
+    });
+
+    if (items.length) break;
+  }
+
+  return items;
+}

package/service/scraperService.js

@@ -0,0 +1,38 @@
+import axios from 'axios';
+import { load } from 'cheerio';
+
+export const defaultHeaders = {
+  'User-Agent': 'Mozilla/5.0 (compatible; ShirayukiBot/1.0; +https://example.com/bot)',
+  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+  'Accept-Language': 'en-US,en;q=0.5',
+  'Accept-Encoding': 'gzip, deflate',
+  'Connection': 'keep-alive',
+  'Upgrade-Insecure-Requests': '1'
+};
+
+export function resolveUrlFactory(base) {
+  return (u) => {
+    if (!u) return null;
+    try {
+      return new URL(u, base).href;
+    } catch (e) {
+      if (u.startsWith('//')) return 'https:' + u;
+      if (u.startsWith('/')) return base + u;
+      return u;
+    }
+  };
+}
+
+export async function fetchAndLoad(url) {
+  const resp = await axios.get(url, { 
+    headers: defaultHeaders, 
+    timeout: 5000, // Reduced to 5 seconds for production
+    maxRedirects: 2, // Reduce redirects further
+    validateStatus: function (status) {
+      return status >= 200 && status < 300; // Only accept 2xx status codes
+    }
+  });
+  return load(resp.data);
+}
+
+export default { fetchAndLoad, resolveUrlFactory, defaultHeaders };