ytstats 0.1.0

package/src/api/analytics.js

@@ -0,0 +1,259 @@
+import { call } from './client.js';
+import { rowsFromAnalytics } from './transforms.js';
+
+/**
+ * YouTube Analytics API v2 (reports.query, ids=channel==MINE).
+ *
+ * Two API limits are encoded here rather than left to callers, because exceeding
+ * either produces an opaque failure:
+ *   - dimensions=video rejects maxResults above 200
+ *   - insightTrafficSourceDetail requires maxResults and errors above ~25, and
+ *     only tolerates the `views` metric
+ */
+const MAX_VIDEO_ROWS = 200;
+const MAX_DETAIL_ROWS = 25;
+
+async function query(apis, { startDate, endDate, metrics, dimensions, filters, sort, maxResults }) {
+  const params = { ids: 'channel==MINE', startDate, endDate, metrics };
+  if (dimensions) params.dimensions = dimensions;
+  if (filters) params.filters = filters;
+  if (sort) params.sort = sort;
+  if (maxResults) params.maxResults = maxResults;
+
+  const res = await call(() => apis.analytics.reports.query(params));
+  return res.data;
+}
+
+const num = v => (v === undefined || v === null ? null : v);
+
+export async function fetchDailyAnalytics(apis, { startDate, endDate }) {
+  const data = await query(apis, {
+    startDate,
+    endDate,
+    // Thumbnail impressions/CTR are documented for this API but always fail;
+    // reach data comes from the Reporting API instead.
+    metrics: 'views,estimatedMinutesWatched,averageViewDuration,likes,dislikes,comments,shares,subscribersGained,subscribersLost',
+    dimensions: 'day',
+    sort: 'day',
+  });
+
+  return rowsFromAnalytics(data).map(r => ({
+    date: r.day,
+    views: num(r.views),
+    estimatedMinutesWatched: num(r.estimatedMinutesWatched),
+    averageViewDuration: num(r.averageViewDuration),
+    likes: num(r.likes),
+    dislikes: num(r.dislikes),
+    comments: num(r.comments),
+    shares: num(r.shares),
+    subscribersGained: num(r.subscribersGained),
+    subscribersLost: num(r.subscribersLost),
+  }));
+}
+
+/** Card/annotation metrics. Degrades to [] — some channels never have this data. */
+export async function fetchCardMetrics(apis, { startDate, endDate }) {
+  try {
+    const data = await query(apis, {
+      startDate,
+      endDate,
+      metrics: 'views,annotationClickThroughRate,cardClicks,cardImpressions',
+      dimensions: 'day',
+      sort: 'day',
+    });
+    return rowsFromAnalytics(data).map(r => ({
+      date: r.day,
+      annotationClickThroughRate: num(r.annotationClickThroughRate),
+      cardClicks: num(r.cardClicks),
+      cardImpressions: num(r.cardImpressions),
+    }));
+  } catch {
+    return [];
+  }
+}
+
+export async function fetchVideoAnalytics(apis, { startDate, endDate, maxResults = MAX_VIDEO_ROWS }) {
+  const data = await query(apis, {
+    startDate,
+    endDate,
+    metrics: 'views,estimatedMinutesWatched,averageViewDuration,averageViewPercentage,likes,comments,shares,subscribersGained,subscribersLost',
+    dimensions: 'video',
+    sort: '-views',
+    maxResults: Math.min(maxResults, MAX_VIDEO_ROWS),
+  });
+
+  return rowsFromAnalytics(data).map(r => ({
+    videoId: r.video,
+    views: num(r.views),
+    estimatedMinutesWatched: num(r.estimatedMinutesWatched),
+    averageViewDuration: num(r.averageViewDuration),
+    averageViewPercentage: num(r.averageViewPercentage),
+    likes: num(r.likes),
+    comments: num(r.comments),
+    shares: num(r.shares),
+    subscribersGained: num(r.subscribersGained),
+    subscribersLost: num(r.subscribersLost),
+  }));
+}
+
+export async function fetchTrafficSources(apis, { startDate, endDate }) {
+  const data = await query(apis, {
+    startDate,
+    endDate,
+    metrics: 'views,estimatedMinutesWatched',
+    dimensions: 'insightTrafficSourceType',
+    sort: '-views',
+  });
+  return rowsFromAnalytics(data).map(r => ({
+    sourceType: r.insightTrafficSourceType,
+    views: num(r.views),
+    estimatedMinutesWatched: num(r.estimatedMinutesWatched),
+  }));
+}
+
+export async function fetchDemographics(apis, { startDate, endDate }) {
+  const data = await query(apis, {
+    startDate,
+    endDate,
+    metrics: 'viewerPercentage',
+    dimensions: 'ageGroup,gender',
+  });
+  return rowsFromAnalytics(data).map(r => ({
+    ageGroup: r.ageGroup,
+    gender: r.gender,
+    viewerPercentage: num(r.viewerPercentage),
+  }));
+}
+
+export async function fetchDeviceTypes(apis, { startDate, endDate }) {
+  const data = await query(apis, {
+    startDate,
+    endDate,
+    metrics: 'views,estimatedMinutesWatched',
+    dimensions: 'deviceType',
+    sort: '-views',
+  });
+  return rowsFromAnalytics(data).map(r => ({
+    deviceType: r.deviceType,
+    views: num(r.views),
+    estimatedMinutesWatched: num(r.estimatedMinutesWatched),
+  }));
+}
+
+/**
+ * Shorts vs long-form vs live, using YouTube's own creatorContentType.
+ * Values are lowercase here (`shorts`, `videoOnDemand`) whereas the duration-based
+ * `contentType` on a video resource is uppercase (`SHORTS`). They can disagree.
+ */
+export async function fetchContentTypes(apis, { startDate, endDate }) {
+  const data = await query(apis, {
+    startDate,
+    endDate,
+    metrics: 'views,estimatedMinutesWatched,likes,shares,subscribersGained,subscribersLost',
+    dimensions: 'creatorContentType',
+    sort: '-views',
+  });
+  return rowsFromAnalytics(data).map(r => ({
+    contentType: r.creatorContentType,
+    views: num(r.views),
+    estimatedMinutesWatched: num(r.estimatedMinutesWatched),
+    likes: num(r.likes),
+    shares: num(r.shares),
+    subscribersGained: num(r.subscribersGained),
+    subscribersLost: num(r.subscribersLost),
+  }));
+}
+
+export async function fetchSearchTerms(apis, { startDate, endDate, maxResults = MAX_DETAIL_ROWS }) {
+  const data = await query(apis, {
+    startDate,
+    endDate,
+    metrics: 'views',
+    dimensions: 'insightTrafficSourceDetail',
+    filters: 'insightTrafficSourceType==YT_SEARCH',
+    sort: '-views',
+    maxResults: Math.min(maxResults, MAX_DETAIL_ROWS),
+  });
+  return rowsFromAnalytics(data).map(r => ({
+    searchTerm: r.insightTrafficSourceDetail,
+    views: num(r.views),
+  }));
+}
+
+export async function fetchGeography(apis, { startDate, endDate, maxResults = 50 }) {
+  const data = await query(apis, {
+    startDate,
+    endDate,
+    metrics: 'views,estimatedMinutesWatched,subscribersGained,subscribersLost',
+    dimensions: 'country',
+    sort: '-views',
+    maxResults,
+  });
+  return rowsFromAnalytics(data).map(r => ({
+    country: r.country,
+    views: num(r.views),
+    estimatedMinutesWatched: num(r.estimatedMinutesWatched),
+    subscribersGained: num(r.subscribersGained),
+    subscribersLost: num(r.subscribersLost),
+  }));
+}
+
+export async function fetchPlaybackLocations(apis, { startDate, endDate }) {
+  const data = await query(apis, {
+    startDate,
+    endDate,
+    metrics: 'views,estimatedMinutesWatched',
+    dimensions: 'insightPlaybackLocationType',
+    sort: '-views',
+  });
+  return rowsFromAnalytics(data).map(r => ({
+    locationType: r.insightPlaybackLocationType,
+    views: num(r.views),
+    estimatedMinutesWatched: num(r.estimatedMinutesWatched),
+  }));
+}
+
+export async function fetchTrafficSourceDetails(apis, { startDate, endDate, sourceType, maxResults = MAX_DETAIL_ROWS }) {
+  const data = await query(apis, {
+    startDate,
+    endDate,
+    // estimatedMinutesWatched triggers an internal error on this dimension.
+    metrics: 'views',
+    dimensions: 'insightTrafficSourceDetail',
+    filters: `insightTrafficSourceType==${sourceType}`,
+    sort: '-views',
+    maxResults: Math.min(maxResults, MAX_DETAIL_ROWS),
+  });
+  return rowsFromAnalytics(data).map(r => ({
+    sourceType,
+    detail: r.insightTrafficSourceDetail,
+    views: num(r.views),
+  }));
+}
+
+/**
+ * Retention curve for one video, ~100 points at 1% intervals.
+ * Ratios above 1.0 are normal for Shorts (viewers loop) and are never clamped.
+ */
+export async function fetchAudienceRetention(apis, { videoId, startDate, endDate }) {
+  const data = await query(apis, {
+    startDate,
+    endDate,
+    metrics: 'audienceWatchRatio',
+    dimensions: 'elapsedVideoTimeRatio',
+    filters: `video==${videoId}`,
+  });
+  return rowsFromAnalytics(data).map(r => ({
+    position: r.elapsedVideoTimeRatio,
+    ratio: r.audienceWatchRatio,
+  }));
+}
+
+/** Escape hatch: arbitrary metric/dimension combination, columns included. */
+export async function runCustomReport(apis, opts) {
+  const data = await query(apis, opts);
+  return {
+    columns: data.columnHeaders?.map(h => ({ name: h.name, type: h.dataType })) ?? [],
+    rows: rowsFromAnalytics(data),
+  };
+}

package/src/api/client.js

@@ -0,0 +1,40 @@
+import { google } from 'googleapis';
+import { mapGoogleError } from '../errors.js';
+
+/**
+ * Bundle the three YouTube API surfaces plus an authenticated CSV downloader.
+ *
+ * Every fetcher takes this bundle as its first argument, so tests hand in plain
+ * objects and never touch the network.
+ */
+export function createApis(authClient) {
+  return {
+    youtube: google.youtube({ version: 'v3', auth: authClient }),
+    analytics: google.youtubeAnalytics({ version: 'v2', auth: authClient }),
+    reporting: google.youtubereporting({ version: 'v1', auth: authClient }),
+
+    /** Reporting API report bodies are plain CSV behind an OAuth-protected URL. */
+    async downloadCsv(url) {
+      const token = await authClient.getAccessToken();
+      const res = await fetch(url, {
+        headers: {
+          Authorization: `Bearer ${token?.token ?? token}`,
+          'Accept-Encoding': 'gzip',
+        },
+      });
+      if (!res.ok) {
+        throw new Error(`Failed to download report (${res.status} ${res.statusText})`);
+      }
+      return res.text();
+    },
+  };
+}
+
+/** Run an API call, converting Google's error shapes into typed YtStatsErrors. */
+export async function call(fn) {
+  try {
+    return await fn();
+  } catch (err) {
+    throw mapGoogleError(err);
+  }
+}

package/src/api/data.js

@@ -0,0 +1,65 @@
+import { call } from './client.js';
+import { normalizeChannel, normalizeVideo } from './transforms.js';
+
+const BATCH_SIZE = 50;
+
+/** The authenticated user's channel, or null when the account owns none. */
+export async function fetchChannel(apis) {
+  const res = await call(() =>
+    apis.youtube.channels.list({
+      part: 'snippet,contentDetails,statistics,status,topicDetails',
+      mine: true,
+    }),
+  );
+  const channel = res.data.items?.[0];
+  return channel ? normalizeChannel(channel) : null;
+}
+
+/**
+ * Every video id on the channel, via the uploads playlist.
+ *
+ * playlistItems.list costs 1 quota unit per page of 50; search.list would cost
+ * 100 per call. Do not swap this for search.
+ */
+export async function fetchAllVideoIds(apis, uploadsPlaylistId) {
+  if (!uploadsPlaylistId) return [];
+
+  const ids = [];
+  let pageToken;
+
+  do {
+    const res = await call(() =>
+      apis.youtube.playlistItems.list({
+        part: 'contentDetails',
+        playlistId: uploadsPlaylistId,
+        maxResults: BATCH_SIZE,
+        pageToken,
+      }),
+    );
+    for (const item of res.data.items ?? []) {
+      const id = item.contentDetails?.videoId;
+      if (id) ids.push(id);
+    }
+    pageToken = res.data.nextPageToken;
+  } while (pageToken);
+
+  return ids;
+}
+
+/** Full video resources for the given ids, batched 50 at a time (1 unit per batch). */
+export async function fetchVideos(apis, videoIds) {
+  if (!videoIds?.length) return [];
+
+  const videos = [];
+  for (let i = 0; i < videoIds.length; i += BATCH_SIZE) {
+    const batch = videoIds.slice(i, i + BATCH_SIZE);
+    const res = await call(() =>
+      apis.youtube.videos.list({
+        part: 'snippet,contentDetails,statistics,status,liveStreamingDetails,topicDetails',
+        id: batch.join(','),
+      }),
+    );
+    videos.push(...(res.data.items ?? []).map(normalizeVideo));
+  }
+  return videos;
+}

package/src/api/reporting.js

@@ -0,0 +1,100 @@
+import { call } from './client.js';
+import { parseCsv, normalizeReportingDate } from './transforms.js';
+
+/**
+ * YouTube Reporting API v1.
+ *
+ * This is the only source of thumbnail impressions and CTR — the equivalent
+ * Analytics API metrics are documented but always return "query is not supported"
+ * (https://issuetracker.google.com/issues/254665034).
+ *
+ * It is asynchronous by design: create a job, YouTube generates daily CSVs, we
+ * download them. First run therefore returns nothing for 24-48h, and the data is
+ * always 1-2 days behind — the same lag YouTube Studio shows.
+ */
+export const REACH_REPORT_TYPE = 'channel_reach_basic_a1';
+const REACH_JOB_NAME = 'ytstats channel reach (CTR)';
+
+export async function listReachJobs(apis) {
+  const res = await call(() => apis.reporting.jobs.list());
+  return res.data.jobs ?? [];
+}
+
+/** Find or create the reach job. Safe to call on every run. */
+export async function ensureReachJob(apis) {
+  const jobs = await listReachJobs(apis);
+  const existing = jobs.find(j => j.reportTypeId === REACH_REPORT_TYPE);
+  if (existing) return existing;
+
+  const res = await call(() =>
+    apis.reporting.jobs.create({
+      requestBody: { reportTypeId: REACH_REPORT_TYPE, name: REACH_JOB_NAME },
+    }),
+  );
+  return res.data;
+}
+
+async function listReports(apis, jobId) {
+  const reports = [];
+  let pageToken;
+  do {
+    const params = { jobId };
+    if (pageToken) params.pageToken = pageToken;
+    const res = await call(() => apis.reporting.jobs.reports.list(params));
+    reports.push(...(res.data.reports ?? []));
+    pageToken = res.data.nextPageToken;
+  } while (pageToken);
+  return reports;
+}
+
+/**
+ * Download every available reach report and merge them.
+ *
+ * Reports overlap, so rows are deduped on (date, videoId) with the last write
+ * winning — later reports carry corrected figures for the same day.
+ */
+export async function fetchReach(apis, { onProgress } = {}) {
+  const job = await ensureReachJob(apis);
+  const reports = await listReports(apis, job.id);
+
+  if (reports.length === 0) {
+    return {
+      job: { id: job.id, reportTypeId: job.reportTypeId, createTime: job.createTime ?? null },
+      reportCount: 0,
+      pending: true,
+      message:
+        'Reporting job is set up. YouTube generates the first reports within 24-48 hours ' +
+        '(including a 30-day backfill). Re-run `ytstats reach` after that.',
+      rows: [],
+    };
+  }
+
+  const deduped = new Map();
+  for (const [i, report] of reports.entries()) {
+    onProgress?.(`Downloading reach report ${i + 1}/${reports.length}...`);
+    const csv = await apis.downloadCsv(report.downloadUrl);
+    for (const row of parseCsv(csv)) {
+      const date = normalizeReportingDate(row.date);
+      const videoId = row.video_id ?? null;
+      deduped.set(`${date}|${videoId}`, {
+        date,
+        channelId: row.channel_id ?? null,
+        videoId,
+        impressions: row.impressions ?? null,
+        // Decimal fraction, not a percentage: 0.0561 means 5.61%.
+        impressionsCtr: row.impressions_ctr ?? null,
+      });
+    }
+  }
+
+  const rows = Array.from(deduped.values()).sort((a, b) =>
+    a.date === b.date ? String(a.videoId).localeCompare(String(b.videoId)) : a.date < b.date ? -1 : 1,
+  );
+
+  return {
+    job: { id: job.id, reportTypeId: job.reportTypeId, createTime: job.createTime ?? null },
+    reportCount: reports.length,
+    pending: false,
+    rows,
+  };
+}

package/src/api/transforms.js

@@ -0,0 +1,170 @@
+/**
+ * Pure shaping helpers. No network, no auth — everything here is deterministic so
+ * it can be tested directly.
+ *
+ * ytstats emits idiomatic camelCase JSON. Mapping that onto any particular
+ * database schema is the consumer's job, not this package's.
+ */
+
+const DURATION_RE = /^P(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+(?:\.\d+)?)S)?)?$/;
+
+/** ISO 8601 duration (PT15M33S, P1DT2H) to whole seconds. 0 when absent/invalid. */
+export function parseDuration(iso) {
+  if (!iso || typeof iso !== 'string') return 0;
+  const m = iso.match(DURATION_RE);
+  if (!m) return 0;
+  const [, d, h, min, s] = m;
+  return (
+    Number(d || 0) * 86400 +
+    Number(h || 0) * 3600 +
+    Number(min || 0) * 60 +
+    Math.floor(Number(s || 0))
+  );
+}
+
+/**
+ * Duration-based content classification.
+ *
+ * Deliberately identical to the historical behaviour: <=60s is a Short. YouTube's
+ * own creatorContentType uses extra signals and can disagree — a 62s video meant
+ * as a Short lands in VIDEO_ON_DEMAND here. Callers wanting YouTube's own view
+ * should read the contentTypes analytics dimension instead.
+ */
+export function classifyContent(video) {
+  if (video?.liveStreamingDetails) return 'LIVE_STREAM';
+  const seconds = parseDuration(video?.contentDetails?.duration);
+  if (seconds > 0 && seconds <= 60) return 'SHORTS';
+  return 'VIDEO_ON_DEMAND';
+}
+
+/**
+ * Minimal RFC 4180 CSV parser: quoted fields, embedded commas/newlines, and ""
+ * escapes. The Reporting API emits video titles and search terms, both of which
+ * routinely contain commas, so naive splitting silently corrupts rows.
+ */
+export function parseCsv(text) {
+  if (typeof text !== 'string' || text.trim() === '') return [];
+
+  const records = [];
+  let field = '';
+  let record = [];
+  let inQuotes = false;
+
+  for (let i = 0; i < text.length; i++) {
+    const ch = text[i];
+
+    if (inQuotes) {
+      if (ch === '"') {
+        if (text[i + 1] === '"') { field += '"'; i++; }
+        else inQuotes = false;
+      } else field += ch;
+      continue;
+    }
+
+    if (ch === '"') { inQuotes = true; continue; }
+    if (ch === ',') { record.push(field); field = ''; continue; }
+    if (ch === '\r') continue;
+    if (ch === '\n') { record.push(field); records.push(record); record = []; field = ''; continue; }
+    field += ch;
+  }
+  if (field !== '' || record.length) { record.push(field); records.push(record); }
+
+  if (records.length < 2) return [];
+
+  const headers = records[0].map(h => h.trim());
+  return records.slice(1)
+    .filter(r => r.some(v => v !== ''))
+    .map(values => {
+      const row = {};
+      headers.forEach((h, i) => { row[h] = coerce(values[i]); });
+      return row;
+    });
+}
+
+/**
+ * Numeric-looking cells become numbers, except values with a leading zero, which
+ * are identifiers (video ids, zero-padded codes) that must stay strings.
+ */
+function coerce(value) {
+  if (value === undefined || value === '') return null;
+  const v = String(value).trim();
+  if (v === '') return null;
+  if (/^-?(0|[1-9]\d*)(\.\d+)?$/.test(v)) return Number(v);
+  return v;
+}
+
+/** Reporting API dates arrive as 20260328.0; everything else uses ISO. Unify. */
+export function normalizeReportingDate(value) {
+  if (value === null || value === undefined) return value ?? null;
+  const s = String(value).trim();
+  if (/^\d{4}-\d{2}-\d{2}$/.test(s)) return s;
+  const m = s.match(/^(\d{4})(\d{2})(\d{2})(?:\.0+)?$/);
+  return m ? `${m[1]}-${m[2]}-${m[3]}` : value;
+}
+
+/** Analytics responses are columnHeaders + positional rows; zip into objects. */
+export function rowsFromAnalytics(data) {
+  if (!data?.rows?.length || !data?.columnHeaders?.length) return [];
+  const headers = data.columnHeaders.map(h => h.name);
+  return data.rows.map(row => {
+    const obj = {};
+    headers.forEach((h, i) => { obj[h] = row[i]; });
+    return obj;
+  });
+}
+
+const int = v => {
+  const n = parseInt(v, 10);
+  return Number.isFinite(n) ? n : 0;
+};
+
+export function normalizeChannel(channel) {
+  const s = channel?.snippet ?? {};
+  const cd = channel?.contentDetails ?? {};
+  const st = channel?.statistics ?? {};
+
+  return {
+    id: channel?.id ?? null,
+    title: s.title ?? null,
+    description: s.description ?? null,
+    customUrl: s.customUrl ?? null,
+    publishedAt: s.publishedAt ?? null,
+    country: s.country ?? null,
+    // YouTube rounds this to 3 significant figures above 1,000 subscribers.
+    subscriberCount: int(st.subscriberCount),
+    viewCount: int(st.viewCount),
+    videoCount: int(st.videoCount),
+    uploadsPlaylistId: cd.relatedPlaylists?.uploads ?? null,
+    thumbnailUrl: s.thumbnails?.high?.url ?? s.thumbnails?.default?.url ?? null,
+  };
+}
+
+export function normalizeVideo(video) {
+  const s = video?.snippet ?? {};
+  const cd = video?.contentDetails ?? {};
+  const st = video?.statistics ?? {};
+  const status = video?.status ?? {};
+
+  return {
+    id: video?.id ?? null,
+    channelId: s.channelId ?? null,
+    title: s.title ?? null,
+    description: s.description ?? null,
+    publishedAt: s.publishedAt ?? null,
+    thumbnailUrl:
+      s.thumbnails?.maxres?.url ?? s.thumbnails?.high?.url ?? s.thumbnails?.default?.url ?? null,
+    duration: cd.duration ?? null,
+    durationSeconds: parseDuration(cd.duration),
+    definition: cd.definition ?? null,
+    dimension: cd.dimension ?? null,
+    caption: cd.caption === 'true' || cd.caption === true,
+    tags: Array.isArray(s.tags) ? s.tags : [],
+    categoryId: s.categoryId ?? null,
+    privacyStatus: status.privacyStatus ?? null,
+    madeForKids: Boolean(status.madeForKids),
+    viewCount: int(st.viewCount),
+    likeCount: int(st.likeCount),
+    commentCount: int(st.commentCount),
+    contentType: classifyContent(video),
+  };
+}