vlr-stats-scraper 1.3.0 → 1.4.1

package/main.mjs

@@ -1,86 +1,91 @@
-#!/usr/bin/env node
-import promptSync from 'prompt-sync';
-import { scrapeWebsite } from './scripts/player-link-scraper.mjs';
-import scrapePlayer from './scripts/player-info-scraper.mjs';
-import {
-  savePlayer,
-  saveEvent,
-  addPlayerToEvent,
-  getEventByName,
-  emptyEvent,
-  clearEvent,
-  clearDatabase,
-} from './scripts/db.mjs';
-
-const args = process.argv.slice(2);
-
-if (args[0] === '--clear') {
-  const prompt = promptSync();
-  const answer = prompt('This deletes every event and player in players.db. Type "yes" to continue: ');
-  if (answer?.trim().toLowerCase() === 'yes') {
-    clearDatabase();
-    console.log('Cleared players.db');
-  } else {
-    console.log('Cancelled');
-  }
-  process.exit(0);
-}
-
-if (args[0] === '--clear-event') {
-  const name = args[1];
-  if (!name) {
-    console.error('Usage: valstats --clear-event <event-name>');
-    process.exit(1);
-  }
-  if (clearEvent(name)) {
-    console.log(`Removed "${name}" and its players from players.db`);
-  } else {
-    console.error(`No event named "${name}" in players.db`);
-    process.exit(1);
-  }
-  process.exit(0);
-}
-
-const [link, year, eventName] = args;
-
-if (!link || !year || !eventName) {
-  console.error('Usage: valstats <link> <year> <event-name>');
-  console.error('       valstats --clear-event <event-name>');
-  console.error('       valstats --clear');
-  process.exit(1);
-}
-
-const isStage = eventName.toLowerCase().includes('stage');
-
-let region = null;
-if (isStage) {
-  const prompt = promptSync();
-  region = prompt('Enter the region: ');
-}
-
-// Re-adding an event + year that's already saved replaces its old players.
-const existing = getEventByName(eventName);
-if (existing && existing.year === parseInt(year, 10)) {
-  emptyEvent(eventName);
-  console.log(`"${eventName}" (${year}) is already in players.db, replacing it`);
-}
-
-const eventId = saveEvent({
-  name: eventName,
-  url: link,
-  region,
-  year: parseInt(year, 10),
-  stage: null,
-});
-
-const links = await scrapeWebsite(link);
-
-for (const playerLink of links) {
-  const player = await scrapePlayer(playerLink, region);
-  if (player) {
-    savePlayer(player);
-    addPlayerToEvent(eventId, player.name);
-    console.log(`Saved ${player.name} to players.db under "${eventName}"`);
-    console.log(player);
-  }
-}
+#!/usr/bin/env node
+import promptSync from 'prompt-sync';
+import { scrapeWebsite } from './scripts/player-link-scraper.mjs';
+import scrapePlayer from './scripts/player-info-scraper.mjs';
+import {
+  savePlayer,
+  saveEvent,
+  addPlayerToEvent,
+  getEventByName,
+  emptyEvent,
+  clearEvent,
+  clearDatabase,
+} from './scripts/db.mjs';
+
+const args = process.argv.slice(2);
+
+if (args[0] === '--clear') {
+  const prompt = promptSync();
+  const answer = prompt('This deletes every event and player in players.db. Type "yes" to continue: ');
+  if (answer?.trim().toLowerCase() === 'yes') {
+    clearDatabase();
+    console.log('Cleared players.db');
+  } else {
+    console.log('Cancelled');
+  }
+  process.exit(0);
+}
+
+if (args[0] === '--clear-event') {
+  const name = args[1];
+  if (!name) {
+    console.error('Usage: valstats --clear-event <event-name>');
+    process.exit(1);
+  }
+  if (clearEvent(name)) {
+    console.log(`Removed "${name}" and its players from players.db`);
+  } else {
+    console.error(`No event named "${name}" in players.db`);
+    process.exit(1);
+  }
+  process.exit(0);
+}
+
+const [link, year, eventName] = args;
+
+if (!link || !year || !eventName) {
+  console.error('Usage: valstats <link> <year> <event-name>');
+  console.error('       valstats --clear-event <event-name>');
+  console.error('       valstats --clear');
+  process.exit(1);
+}
+
+const isStage = eventName.toLowerCase().includes('stage');
+
+let region = null;
+if (isStage) {
+  const prompt = promptSync();
+  region = prompt('Enter the region: ');
+}
+
+// Re-adding an event + year that's already saved replaces its old players.
+const existing = getEventByName(eventName);
+if (existing && existing.year === parseInt(year, 10)) {
+  emptyEvent(eventName);
+  console.log(`"${eventName}" (${year}) is already in players.db, replacing it`);
+}
+
+const eventId = saveEvent({
+  name: eventName,
+  url: link,
+  region,
+  year: parseInt(year, 10),
+  stage: null,
+});
+
+const links = await scrapeWebsite(link);
+
+if (!links) {
+  console.error(`Couldn't get the player list from ${link}`);
+  process.exit(1);
+}
+
+for (const playerLink of links) {
+  const player = await scrapePlayer(playerLink, region);
+  if (player) {
+    savePlayer(player);
+    addPlayerToEvent(eventId, player.name);
+    console.log(`Saved ${player.name} to players.db under "${eventName}"`);
+    console.log(player);
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vlr-stats-scraper",
-  "version": "1.3.0",
+  "version": "1.4.1",
   "description": "Scrapes player stats from vlr.gg and stores them in a local SQLite database, grouped by tournament or stage.",
   "homepage": "https://github.com/OverYander/VLR-Webscraper#readme",
   "bugs": {

package/scripts/player-info-scraper.mjs

@@ -1,8 +1,8 @@
-import * as cheerio from 'cheerio';
+import * as cheerio from 'cheerio'
 import { Hyperbrowser } from "@hyperbrowser/sdk"
-import { config } from "dotenv";
+import { config } from "dotenv"
 
-config();
+config()
 
 let client;
 function getClient() {
@@ -25,15 +25,15 @@ async function scrapePlayer(url, region) {
     })
 
     const $ = cheerio.load(scrapeData.data.html);
-
     const agentsPlayed = findAgents($)
+
     return {
       name: getPlayerName(url),
       team: getCurTeam($),
       past_teams: getPastTeam($),
       winnings: getWinnings($),
       most_played_agents: agentsPlayed.map(data => data.Agent),
-      avg_kda: agentsPlayed.reduce((acc, data) => acc + (parseFloat(data['K:D'])) / agentsPlayed.length, 0),
+      avg_kda: agentsPlayed.reduce((acc, data) => acc + (parseFloat(data['K:D'])) / agentsPlayed.length, 0).toFixed(2),
       nationality: getNationality($),
       region: region,
       stats: agentsPlayed
@@ -44,15 +44,16 @@ async function scrapePlayer(url, region) {
   }
 }
 
-export default scrapePlayer;
-
+// testing
 /*
 node scripts/player-info-scraper.mjs
 */
 
+scrapePlayer("https://www.vlr.gg/player/939/mwzera/?timespan=all", "AMER")
+
+
 function getPlayerName(link) {
-  let name = link.substring(link.lastIndexOf('/') + 1)
-  console.log(name)
+  let name = new URL(link).pathname.split('/').filter(ele => ele !== '').pop()
   return name
 }
 
@@ -66,8 +67,9 @@ function findAgents($) {
   const agentStats = header.next().children()
   const agents = agentStats.map((i, el) => {
     el = $(el)
+    // console.log('agent found:', el.children().first().find('img').attr('alt'))?.trim()
     return {
-      agent: el.children().first().html().trim(),
+      agent: el.children().first().find('img').attr('alt')?.trim(),
       stats: el.text().split('\t').map(ele => ele.trim()).filter(ele => ele !== '')
     }
   }).get()
@@ -91,7 +93,8 @@ function getCurTeam($) {
   // console.log('getCurTeam ran')
   const currentTeam = $('h2:contains("Current Teams")').first().next().text().trim()
   const currentTeamName = currentTeam.split('\t').filter(ele => ele.trim() !== '')
-  return currentTeamName[0].trim()
+  // Free agents have no current team
+  return currentTeamName.length ? currentTeamName[0].trim() : null
 }
 
 function getPastTeam($) {
@@ -120,4 +123,6 @@ function getNationality($) {
   // console.log('flags found:', $('.flag').length);
   // console.log(header.parent().text().trim())
   return header.parent().text().trim()
-}
+}
+
+export default scrapePlayer