vlr-stats-scraper 1.2.0 → 1.4.0

package/main.mjs

@@ -1,86 +1,91 @@
-#!/usr/bin/env node
-import promptSync from 'prompt-sync';
-import { scrapeWebsite } from './scripts/player-link-scraper.mjs';
-import scrapePlayer from './scripts/player-info-scraper.mjs';
-import {
-  savePlayer,
-  saveEvent,
-  addPlayerToEvent,
-  getEventByName,
-  emptyEvent,
-  clearEvent,
-  clearDatabase,
-} from './scripts/db.mjs';
-
-const args = process.argv.slice(2);
-
-if (args[0] === '--clear') {
-  const prompt = promptSync();
-  const answer = prompt('This deletes every event and player in players.db. Type "yes" to continue: ');
-  if (answer?.trim().toLowerCase() === 'yes') {
-    clearDatabase();
-    console.log('Cleared players.db');
-  } else {
-    console.log('Cancelled');
-  }
-  process.exit(0);
-}
-
-if (args[0] === '--clear-event') {
-  const name = args[1];
-  if (!name) {
-    console.error('Usage: valstats --clear-event <event-name>');
-    process.exit(1);
-  }
-  if (clearEvent(name)) {
-    console.log(`Removed "${name}" and its players from players.db`);
-  } else {
-    console.error(`No event named "${name}" in players.db`);
-    process.exit(1);
-  }
-  process.exit(0);
-}
-
-const [link, year, eventName] = args;
-
-if (!link || !year || !eventName) {
-  console.error('Usage: valstats <link> <year> <event-name>');
-  console.error('       valstats --clear-event <event-name>');
-  console.error('       valstats --clear');
-  process.exit(1);
-}
-
-const isStage = eventName.toLowerCase().includes('stage');
-
-let region = null;
-if (isStage) {
-  const prompt = promptSync();
-  region = prompt('Enter the region: ');
-}
-
-// Re-adding an event + year that's already saved replaces its old players.
-const existing = getEventByName(eventName);
-if (existing && existing.year === parseInt(year, 10)) {
-  emptyEvent(eventName);
-  console.log(`"${eventName}" (${year}) is already in players.db, replacing it`);
-}
-
-const eventId = saveEvent({
-  name: eventName,
-  url: link,
-  region,
-  year: parseInt(year, 10),
-  stage: null,
-});
-
-const links = await scrapeWebsite(link);
-
-for (const playerLink of links) {
-  const player = await scrapePlayer(playerLink, region);
-  if (player) {
-    savePlayer(player);
-    addPlayerToEvent(eventId, player.name);
-    console.log(`Saved ${player.name} to players.db under "${eventName}"`);
-    console.log(player);
-  }
-}
+#!/usr/bin/env node
+import promptSync from 'prompt-sync';
+import { scrapeWebsite } from './scripts/player-link-scraper.mjs';
+import scrapePlayer from './scripts/player-info-scraper.mjs';
+import {
+  savePlayer,
+  saveEvent,
+  addPlayerToEvent,
+  getEventByName,
+  emptyEvent,
+  clearEvent,
+  clearDatabase,
+} from './scripts/db.mjs';
+
+const args = process.argv.slice(2);
+
+if (args[0] === '--clear') {
+  const prompt = promptSync();
+  const answer = prompt('This deletes every event and player in players.db. Type "yes" to continue: ');
+  if (answer?.trim().toLowerCase() === 'yes') {
+    clearDatabase();
+    console.log('Cleared players.db');
+  } else {
+    console.log('Cancelled');
+  }
+  process.exit(0);
+}
+
+if (args[0] === '--clear-event') {
+  const name = args[1];
+  if (!name) {
+    console.error('Usage: valstats --clear-event <event-name>');
+    process.exit(1);
+  }
+  if (clearEvent(name)) {
+    console.log(`Removed "${name}" and its players from players.db`);
+  } else {
+    console.error(`No event named "${name}" in players.db`);
+    process.exit(1);
+  }
+  process.exit(0);
+}
+
+const [link, year, eventName] = args;
+
+if (!link || !year || !eventName) {
+  console.error('Usage: valstats <link> <year> <event-name>');
+  console.error('       valstats --clear-event <event-name>');
+  console.error('       valstats --clear');
+  process.exit(1);
+}
+
+const isStage = eventName.toLowerCase().includes('stage');
+
+let region = null;
+if (isStage) {
+  const prompt = promptSync();
+  region = prompt('Enter the region: ');
+}
+
+// Re-adding an event + year that's already saved replaces its old players.
+const existing = getEventByName(eventName);
+if (existing && existing.year === parseInt(year, 10)) {
+  emptyEvent(eventName);
+  console.log(`"${eventName}" (${year}) is already in players.db, replacing it`);
+}
+
+const eventId = saveEvent({
+  name: eventName,
+  url: link,
+  region,
+  year: parseInt(year, 10),
+  stage: null,
+});
+
+const links = await scrapeWebsite(link);
+
+if (!links) {
+  console.error(`Couldn't get the player list from ${link}`);
+  process.exit(1);
+}
+
+for (const playerLink of links) {
+  const player = await scrapePlayer(playerLink, region);
+  if (player) {
+    savePlayer(player);
+    addPlayerToEvent(eventId, player.name);
+    console.log(`Saved ${player.name} to players.db under "${eventName}"`);
+    console.log(player);
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vlr-stats-scraper",
-  "version": "1.2.0",
+  "version": "1.4.0",
   "description": "Scrapes player stats from vlr.gg and stores them in a local SQLite database, grouped by tournament or stage.",
   "homepage": "https://github.com/OverYander/VLR-Webscraper#readme",
   "bugs": {

package/scripts/player-info-scraper.mjs

@@ -15,7 +15,7 @@ function getClient() {
 }
 
 async function scrapePlayer(url, region) {
-  console.log('scrapePlayer ran')
+  // console.log('scrapePlayer ran')
   try {
     const scrapeData = await getClient().scrape.startAndWait({
       url: url,
@@ -51,12 +51,14 @@ node scripts/player-info-scraper.mjs
 */
 
 function getPlayerName(link) {
-  console.log('getPlayerName ran')
-  return link.substring(link.lastIndexOf('/') + 1)
+  // Links look like .../player/123/name/?timespan=all, so take the last path segment
+  let name = new URL(link).pathname.split('/').filter(ele => ele !== '').pop()
+  console.log(name)
+  return name
 }
 
 function findAgents($) {
-  console.log('findAgents ran')
+  // console.log('findAgents ran')
   const header = $('.mod-agent-rows').first().children().first()
 
   const statCategories = header.text().split('\n').map(ele => ele.trim()).filter(ele => ele !== '')
@@ -66,7 +68,7 @@ function findAgents($) {
   const agents = agentStats.map((i, el) => {
     el = $(el)
     return {
-      agent: el.children().first().html().trim(),
+      agent: el.children().first().find('img').attr('alt')?.trim(),
       stats: el.text().split('\t').map(ele => ele.trim()).filter(ele => ele !== '')
     }
   }).get()
@@ -87,14 +89,15 @@ function findAgents($) {
 }
 
 function getCurTeam($) {
-  console.log('getCurTeam ran')
+  // console.log('getCurTeam ran')
   const currentTeam = $('h2:contains("Current Teams")').first().next().text().trim()
   const currentTeamName = currentTeam.split('\t').filter(ele => ele.trim() !== '')
-  return currentTeamName[0].trim()
+  // Free agents have no current team
+  return currentTeamName.length ? currentTeamName[0].trim() : null
 }
 
 function getPastTeam($) {
-    console.log('getPastTeam ran')
+    // console.log('getPastTeam ran')
     const pastTeams = $('h2:contains("Past Teams")').first().next().text().trim()
     const pastTeamNames = pastTeams.split('\t').filter(ele => ele.trim() !== ''
                           && !ele.includes('joined')
@@ -105,18 +108,18 @@ function getPastTeam($) {
 }
 
 function getWinnings($) {
-    console.log('getWinnings ran')
+    // console.log('getWinnings ran')
     const events = $('h2:contains("Event Placements")').first().next()
     const winnings = events.children().first().text().trim()
-    // console.log(winnings.split('\t').map(ele => ele.trim()).filter(ele => ele !== '')[1])
+    // // console.log(winnings.split('\t').map(ele => ele.trim()).filter(ele => ele !== '')[1])
     return winnings.split('\t').map(ele => ele.trim()).filter(ele => ele !== '')[1]
   
 }
 
 function getNationality($) {
-  console.log('getNationality ran')
+  // console.log('getNationality ran')
   const header = $('.flag').first()
-  console.log('flags found:', $('.flag').length);
-  console.log(header.parent().text().trim())
+  // console.log('flags found:', $('.flag').length);
+  // console.log(header.parent().text().trim())
   return header.parent().text().trim()
 }