shirayuki-anime-scraper-api 1.0.0

package/models/Episode.js

@@ -0,0 +1,49 @@
+import mongoose from 'mongoose';
+
+const episodeSchema = new mongoose.Schema({
+    anime_id: {
+        type: String,
+        required: true,
+        index: true
+    },
+    episode_number: {
+        type: String,
+        required: true
+    },
+    episode_url: {
+        type: String,
+        required: true,
+        unique: true
+    },
+    streaming_data: {
+        title: String,
+        episode_number: String,
+        streaming_link: String,
+        range_id: String,
+        all_ranges: [String]
+    },
+    extraction_time_seconds: {
+        type: Number,
+        required: true
+    },
+    last_updated: {
+        type: Date,
+        default: Date.now
+    },
+    cache_expires_at: {
+        type: Date,
+        required: true
+    }
+}, {
+    timestamps: true
+});
+
+// Compound index for faster queries
+episodeSchema.index({ anime_id: 1, episode_number: 1 });
+episodeSchema.index({ episode_url: 1 });
+episodeSchema.index({ cache_expires_at: 1 });
+episodeSchema.index({ last_updated: -1 });
+
+const Episode = mongoose.model('Episode', episodeSchema);
+
+export default Episode;

package/models/Schedule.js

@@ -0,0 +1,50 @@
+import mongoose from 'mongoose';
+
+const scheduleItemSchema = new mongoose.Schema({
+    day: {
+        type: String,
+        required: true,
+        enum: ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday', 'Error']
+    },
+    anime: {
+        type: String,
+        required: true
+    },
+    time: {
+        type: String,
+        required: true
+    }
+}, {
+    timestamps: true
+});
+
+const scheduleSchema = new mongoose.Schema({
+    week_id: {
+        type: String,
+        required: true,
+        unique: true
+    },
+    schedule_data: [scheduleItemSchema],
+    extraction_time_seconds: {
+        type: Number,
+        required: true
+    },
+    total_episodes: {
+        type: Number,
+        default: 0
+    },
+    last_updated: {
+        type: Date,
+        default: Date.now
+    }
+}, {
+    timestamps: true
+});
+
+// Index for faster queries
+scheduleSchema.index({ week_id: 1 });
+scheduleSchema.index({ last_updated: -1 });
+
+const Schedule = mongoose.model('Schedule', scheduleSchema);
+
+export default Schedule;

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "shirayuki-anime-scraper-api",
+  "version": "1.0.0",
+  "description": "A comprehensive anime scraping API that provides anime information, streaming links, and search functionality from HiAnime and other popular anime streaming platforms",
+  "keywords": [
+    "anime",
+    "scraping",
+    "streaming",
+    "api",
+    "hianime",
+    "shirayuki",
+    "anime-api",
+    "scraper"
+  ],
+  "homepage": "https://github.com/Anandadevnath/Shirayuki-Scrapper-API#readme",
+  "bugs": {
+    "url": "https://github.com/Anandadevnath/Shirayuki-Scrapper-API/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Anandadevnath/Shirayuki-Scrapper-API.git"
+  },
+  "license": "ISC",
+  "author": "Anandadevnath",
+  "type": "module",
+  "main": "index.js",
+  "scripts": {
+    "start": "node index.js",
+    "dev": "node index.js",
+    "test": "echo \"Warning: no test specified\" && exit 0"
+  },
+  "dependencies": {
+    "axios": "^1.11.0",
+    "cheerio": "^1.1.2",
+    "dotenv": "^17.0.1",
+    "express": "^5.1.0",
+    "mongodb": "^6.17.0",
+    "mongoose": "^8.16.1",
+    "puppeteer": "^24.20.0",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}

package/routes/anime-list.js

@@ -0,0 +1,67 @@
+import express from 'express';
+import { scrapeAnimeByLetter } from '../scrapeanime/A-Z/AnimeList/filter.js';
+
+const router = express.Router();
+
+router.get('/:letter', async (req, res) => {
+  try {
+    const start = Date.now();
+    const letter = req.params.letter;
+    const page = parseInt(req.query.page) || 1;
+    const axios = (await import('axios')).default;
+    const cheerio = await import('cheerio');
+    const url = `https://123animehub.cc/az-all-anime/${letter}/?page=${page}`;
+    let total_counts = null;
+    try {
+      const { data: html } = await axios.get(url);
+      const $ = cheerio.load(html);
+      const totalText = $('.paging-wrapper .total').first().text().replace(/[^\d]/g, '');
+      if (totalText) total_counts = parseInt(totalText, 10);
+    } catch (e) {
+      total_counts = null;
+    }
+
+    const result = await scrapeAnimeByLetter(letter, page);
+    const duration = (Date.now() - start) / 1000;
+    const indexedResult = result.map((anime, idx) => ({
+      index: idx + 1,
+      ...anime
+    }));
+    res.json({
+      success: true,
+      data: indexedResult,
+      pagination: {
+        current_page: page,
+        total_found: indexedResult.length,
+        total_counts: total_counts,
+        has_next_page: indexedResult.length > 0,
+        has_previous_page: page > 1,
+        next_page: indexedResult.length > 0 ? page + 1 : null,
+        previous_page: page > 1 ? page - 1 : null
+      },
+      extraction_time_seconds: duration,
+      message: `Anime list for letter '${letter}' - Page ${page}`,
+      timestamp: new Date().toISOString(),
+      source_url: url
+    });
+  } catch (error) {
+    const duration = (Date.now() - start) / 1000;
+    res.status(500).json({
+      success: false,
+      error: error.message,
+      extraction_time_seconds: duration,
+      timestamp: new Date().toISOString(),
+      pagination: {
+        current_page: parseInt(req.query.page) || 1,
+        total_found: 0,
+        has_next_page: false,
+        has_previous_page: false,
+        next_page: null,
+        previous_page: null
+      }
+    });
+  }
+});
+
+
+export default router;

package/routes/episodeStream.js

@@ -0,0 +1,64 @@
+import express from 'express';
+import { scrapeSingleEpisode } from '../scrapeanime/SingleEpisode/scrapeSingleEpisode.js';
+
+const router = express.Router();
+
+router.get('/episode-stream', async (req, res) => {
+    try {
+        const animeId = req.query.id;
+        const episodeNumber = req.query.ep;
+
+        if (!animeId || !episodeNumber) {
+            return res.status(400).json({
+                error: 'Both id and ep parameters are required',
+                example: 'http://localhost:5000/episode-stream?id=sentai-daishikkaku-2nd-season-dub&ep=1'
+            });
+        }
+
+        // Validate episode number is numeric
+        if (isNaN(episodeNumber) || episodeNumber < 1) {
+            return res.status(400).json({
+                error: 'Episode number must be a positive integer',
+                example: 'http://localhost:5000/episode-stream?id=anime-name&ep=1'
+            });
+        }
+
+        const episodeUrl = `https://w1.123animes.ru/anime/${animeId}/episode/${episodeNumber}`;
+
+        console.log(`🎯 Fetching streaming link for: ${animeId} Episode ${episodeNumber}`);
+
+        const startTime = Date.now();
+        const result = await scrapeSingleEpisode(episodeUrl);
+        const endTime = Date.now();
+        const duration = (endTime - startTime) / 1000;
+
+        if (result.success) {
+            console.log(`✅ Found streaming link in ${duration.toFixed(2)} seconds`);
+            res.json({
+                success: true,
+                anime_id: animeId,
+                episode: episodeNumber,
+                data: result.data,
+                extraction_time_seconds: duration
+            });
+        } else {
+            console.log(`❌ Failed to find streaming link: ${result.error}`);
+            res.status(404).json({
+                success: false,
+                error: result.error,
+                anime_id: animeId,
+                episode: episodeNumber,
+                extraction_time_seconds: duration
+            });
+        }
+
+    } catch (error) {
+        console.error('❌ Error fetching episode stream:', error.message);
+        res.status(500).json({
+            error: error.message,
+            timestamp: new Date().toISOString()
+        });
+    }
+});
+
+export default router;

package/routes/genre.js

@@ -0,0 +1,67 @@
+import express from 'express';
+import { scrapeAnimeByGenre } from '../scrapeanime/A-Z/Genre/genre.js';
+
+const router = express.Router();
+
+router.get('/:genre', async (req, res) => {
+  try {
+    const start = Date.now();
+    const genre = req.params.genre;
+    const page = parseInt(req.query.page) || 1;
+    const axios = (await import('axios')).default;
+    const cheerio = await import('cheerio');
+    const url = `https://123animehub.cc/genere/${genre}?page=${page}`;
+    let total_counts = null;
+    try {
+      const { data: html } = await axios.get(url);
+      const $ = cheerio.load(html);
+      const totalText = $('.paging-wrapper .total').first().text().replace(/[^\d]/g, '');
+      if (totalText) total_counts = parseInt(totalText, 10);
+    } catch (e) {
+      total_counts = null;
+    }
+
+    const result = await scrapeAnimeByGenre(genre, page);
+    const duration = (Date.now() - start) / 1000;
+    const indexedResult = result.map((anime, idx) => ({
+      index: idx + 1,
+      ...anime
+    }));
+    res.json({
+      success: true,
+      data: indexedResult,
+      pagination: {
+        current_page: page,
+        total_found: indexedResult.length,
+        total_counts: total_counts,
+        has_next_page: indexedResult.length > 0,
+        has_previous_page: page > 1,
+        next_page: indexedResult.length > 0 ? page + 1 : null,
+        previous_page: page > 1 ? page - 1 : null
+      },
+      extraction_time_seconds: duration,
+      message: `Anime list for genre '${genre}' - Page ${page}`,
+      timestamp: new Date().toISOString(),
+      source_url: url
+    });
+  } catch (error) {
+    const duration = (Date.now() - start) / 1000;
+    res.status(500).json({
+      success: false,
+      error: error.message,
+      extraction_time_seconds: duration,
+      timestamp: new Date().toISOString(),
+      pagination: {
+        current_page: parseInt(req.query.page) || 1,
+        total_found: 0,
+        total_counts: 0,
+        has_next_page: false,
+        has_previous_page: false,
+        next_page: null,
+        previous_page: null
+      }
+    });
+  }
+});
+
+export default router;

package/routes/home.js

@@ -0,0 +1,30 @@
+import express from 'express';
+import scrapeHomepage from '../scrapeanime/homepage/scrapehomepage.js';
+
+const router = express.Router();
+
+router.get('/', async (req, res) => {
+  try {
+    const start = Date.now();
+    const result = await scrapeHomepage(true); // Always include details
+    const duration = (Date.now() - start) / 1000;
+
+    if (result.success) {
+      res.json({
+        success: true,
+        data: result.data,
+        extraction_time_seconds: duration,
+      });
+    } else {
+      res.status(502).json({
+        success: false,
+        error: result.error || 'Unknown error',
+        extraction_time_seconds: duration,
+      });
+    }
+  } catch (err) {
+    res.status(500).json({ success: false, error: err.message });
+  }
+});
+
+export default router;

package/routes/monthly.js

@@ -0,0 +1,37 @@
+import express from 'express';
+import { scrapeHiAnimeMonthlyTop10 } from '../scrapeanime/Leaderboard/Monthly/scrapeHiAnimeMonthlyTop10.js';
+
+const router = express.Router();
+
+router.get('/', async (req, res) => {
+  try {
+    const start = Date.now();
+    console.log('📅 Starting HiAnime Monthly Top 10 scraping...');
+    
+    const result = await scrapeHiAnimeMonthlyTop10();
+    const duration = (Date.now() - start) / 1000;
+
+    console.log(`✅ Monthly Top 10 scraping completed in ${duration}s`);
+
+    res.json({
+      success: true,
+      data: result,
+      extraction_time_seconds: duration,
+      message: "Top 10 monthly viewed anime from HiAnime",
+      timestamp: new Date().toISOString()
+    });
+
+  } catch (error) {
+    const duration = (Date.now() - start) / 1000;
+    console.error('❌ Error scraping monthly top 10:', error.message);
+    
+    res.status(500).json({ 
+      success: false, 
+      error: error.message,
+      extraction_time_seconds: duration,
+      timestamp: new Date().toISOString()
+    });
+  }
+});
+
+export default router;

package/routes/schedule.js

@@ -0,0 +1,174 @@
+import express from 'express';
+import scrapeSchedule from '../scrapeanime/Schedule/schedule.js';
+import connectDB from '../config/database.js';
+import Schedule from '../models/Schedule.js';
+
+const router = express.Router();
+
+const getCurrentWeekId = () => {
+    const now = new Date();
+    const year = now.getFullYear();
+    const weekNumber = getWeekNumber(now);
+    return `${year}-W${weekNumber.toString().padStart(2, '0')}`;
+};
+
+const getWeekNumber = (date) => {
+    const firstDayOfYear = new Date(date.getFullYear(), 0, 1);
+    const pastDaysOfYear = (date - firstDayOfYear) / 86400000;
+    return Math.ceil((pastDaysOfYear + firstDayOfYear.getDay() + 1) / 7);
+};
+
+router.get('/', async (req, res) => {
+    const start = Date.now();
+    
+    try {
+        // Connect to MongoDB
+        await connectDB();
+        
+        const currentWeekId = getCurrentWeekId();
+        
+        // Check if we have recent data (less than 6 hours old)
+        const existingSchedule = await Schedule.findOne({
+            week_id: currentWeekId,
+            last_updated: { $gte: new Date(Date.now() - 6 * 60 * 60 * 1000) } // 6 hours ago
+        }).sort({ last_updated: -1 });
+
+        if (existingSchedule) {
+            console.log(`📋 Returning cached schedule data for ${currentWeekId}`);
+            const cleanData = existingSchedule.schedule_data.map(item => ({
+                day: item.day,
+                anime: item.anime,
+                time: item.time
+            }));
+            
+            return res.json({
+                success: true,
+                data: cleanData,
+                extraction_time_seconds: 0.001,
+                cached: true,
+                week_id: currentWeekId,
+                last_updated: existingSchedule.last_updated,
+                total_episodes: existingSchedule.total_episodes
+            });
+        }
+
+        // Scrape fresh data
+        console.log(`🔄 Scraping fresh schedule data for ${currentWeekId}`);
+        const scheduleData = await scrapeSchedule();
+        const duration = (Date.now() - start) / 1000;
+
+        // Store in MongoDB
+        const savedSchedule = await Schedule.findOneAndUpdate(
+            { week_id: currentWeekId },
+            {
+                schedule_data: scheduleData,
+                extraction_time_seconds: duration,
+                total_episodes: scheduleData.length,
+                last_updated: new Date()
+            },
+            { upsert: true, new: true }
+        );
+
+        console.log(`💾 Saved schedule data to MongoDB: ${scheduleData.length} episodes`);
+
+        // Clean up old data (keep only last 4 weeks)
+        const fourWeeksAgo = new Date();
+        fourWeeksAgo.setDate(fourWeeksAgo.getDate() - 28);
+        
+        const deleteResult = await Schedule.deleteMany({
+            last_updated: { $lt: fourWeeksAgo }
+        });
+        
+        if (deleteResult.deletedCount > 0) {
+            console.log(`🧹 Cleaned up ${deleteResult.deletedCount} old schedule records`);
+        }
+
+        res.json({
+            success: true,
+            data: scheduleData,
+            extraction_time_seconds: duration,
+            cached: false,
+            week_id: currentWeekId,
+            total_episodes: scheduleData.length,
+            saved_to_db: true
+        });
+
+    } catch (err) {
+        console.error('❌ Schedule route error:', err);
+        res.status(500).json({
+            success: false,
+            error: err.message,
+            extraction_time_seconds: (Date.now() - start) / 1000
+        });
+    }
+});
+
+// Additional endpoint to get schedule history
+router.get('/history', async (req, res) => {
+    try {
+        await connectDB();
+        
+        const limit = parseInt(req.query.limit) || 10;
+        const scheduleHistory = await Schedule.find({})
+            .select('week_id total_episodes last_updated extraction_time_seconds')
+            .sort({ last_updated: -1 })
+            .limit(limit);
+
+        res.json({
+            success: true,
+            data: scheduleHistory,
+            total_records: scheduleHistory.length
+        });
+    } catch (err) {
+        res.status(500).json({
+            success: false,
+            error: err.message
+        });
+    }
+});
+
+// Endpoint to force refresh schedule data
+router.post('/refresh', async (req, res) => {
+    const start = Date.now();
+    
+    try {
+        await connectDB();
+        
+        console.log('🔄 Force refreshing schedule data...');
+        const scheduleData = await scrapeSchedule();
+        const duration = (Date.now() - start) / 1000;
+        
+        const currentWeekId = getCurrentWeekId();
+        
+        // Update or create new schedule
+        const updatedSchedule = await Schedule.findOneAndUpdate(
+            { week_id: currentWeekId },
+            {
+                schedule_data: scheduleData,
+                extraction_time_seconds: duration,
+                total_episodes: scheduleData.length,
+                last_updated: new Date()
+            },
+            { upsert: true, new: true }
+        );
+
+        console.log(`💾 Force updated schedule data: ${scheduleData.length} episodes`);
+
+        res.json({
+            success: true,
+            data: scheduleData,
+            extraction_time_seconds: duration,
+            week_id: currentWeekId,
+            total_episodes: scheduleData.length,
+            force_refreshed: true
+        });
+    } catch (err) {
+        res.status(500).json({
+            success: false,
+            error: err.message,
+            extraction_time_seconds: (Date.now() - start) / 1000
+        });
+    }
+});
+
+export default router;

package/routes/search.js

@@ -0,0 +1,79 @@
+import express from 'express';
+import { scrapeSearchSuggestions } from '../scrapeanime/Browse/Suggestion/suggestion.js';
+import { scrapeAnimeSearch } from '../scrapeanime/Browse/Search/search.js';
+
+const router = express.Router();
+
+// GET /search?keyword=one%20piece
+router.get('/', async (req, res) => {
+  const start = Date.now();
+  try {
+    const keyword = req.query.keyword || '';
+    
+    if (!keyword) {
+      return res.status(400).json({
+        success: false,
+        error: 'Query parameter "keyword" is required'
+      });
+    }
+
+    const result = await scrapeAnimeSearch(keyword);
+    const duration = (Date.now() - start) / 1000;
+    
+    res.json({
+      success: true,
+      total_results: result.length,
+      data: result,
+      extraction_time_seconds: duration,
+      message: `Search results for "${keyword}"`,
+      timestamp: new Date().toISOString(),
+      source_url: `https://123animehub.cc/search?keyword=${encodeURIComponent(keyword)}`
+    });
+  } catch (error) {
+    const duration = (Date.now() - start) / 1000;
+    res.status(500).json({
+      success: false,
+      error: error.message,
+      extraction_time_seconds: duration,
+      timestamp: new Date().toISOString()
+    });
+  }
+});
+
+// GET /search/suggestions?q=demon%20slayer
+router.get('/suggestions', async (req, res) => {
+  const start = Date.now();
+  try {
+    const query = req.query.q || req.query.query || '';
+    
+    if (!query) {
+      return res.status(400).json({
+        success: false,
+        error: 'Query parameter "q" is required'
+      });
+    }
+
+    const result = await scrapeSearchSuggestions(query);
+    const duration = (Date.now() - start) / 1000;
+    
+    res.json({
+      success: true,
+      total_suggestions: result.length,
+      data: result,
+      extraction_time_seconds: duration,
+      message: `Search suggestions for "${query}"`,
+      timestamp: new Date().toISOString(),
+      source_url: `https://123animehub.cc/search?keyword=${encodeURIComponent(query)}`
+    });
+  } catch (error) {
+    const duration = (Date.now() - start) / 1000;
+    res.status(500).json({
+      success: false,
+      error: error.message,
+      extraction_time_seconds: duration,
+      timestamp: new Date().toISOString()
+    });
+  }
+});
+
+export default router;