youtube-knowledge-mcp 2.2.0 → 3.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +126 -36
- package/dist/brain-schemas.d.ts +10 -0
- package/dist/brain-schemas.d.ts.map +1 -1
- package/dist/harvest-schemas.d.ts +75 -0
- package/dist/harvest-schemas.d.ts.map +1 -0
- package/dist/harvest-schemas.js +111 -0
- package/dist/harvest-schemas.js.map +1 -0
- package/dist/registry/brain-tools.d.ts.map +1 -1
- package/dist/registry/brain-tools.js +16 -0
- package/dist/registry/brain-tools.js.map +1 -1
- package/dist/registry/channel-tools.d.ts.map +1 -1
- package/dist/registry/channel-tools.js +16 -0
- package/dist/registry/channel-tools.js.map +1 -1
- package/dist/registry/harvest-tools.d.ts +10 -0
- package/dist/registry/harvest-tools.d.ts.map +1 -0
- package/dist/registry/harvest-tools.js +109 -0
- package/dist/registry/harvest-tools.js.map +1 -0
- package/dist/registry/health-tools.d.ts.map +1 -1
- package/dist/registry/health-tools.js +18 -0
- package/dist/registry/health-tools.js.map +1 -1
- package/dist/registry/index.d.ts.map +1 -1
- package/dist/registry/index.js +6 -0
- package/dist/registry/index.js.map +1 -1
- package/dist/registry/media-tools.d.ts.map +1 -1
- package/dist/registry/media-tools.js +31 -0
- package/dist/registry/media-tools.js.map +1 -1
- package/dist/registry/store-tools.d.ts +9 -0
- package/dist/registry/store-tools.d.ts.map +1 -0
- package/dist/registry/store-tools.js +59 -0
- package/dist/registry/store-tools.js.map +1 -0
- package/dist/registry/transcript-tools.d.ts +4 -1
- package/dist/registry/transcript-tools.d.ts.map +1 -1
- package/dist/registry/transcript-tools.js +20 -1
- package/dist/registry/transcript-tools.js.map +1 -1
- package/dist/registry/video-tools.d.ts.map +1 -1
- package/dist/registry/video-tools.js +49 -1
- package/dist/registry/video-tools.js.map +1 -1
- package/dist/registry/vision-tools.d.ts +7 -0
- package/dist/registry/vision-tools.d.ts.map +1 -0
- package/dist/registry/vision-tools.js +55 -0
- package/dist/registry/vision-tools.js.map +1 -0
- package/dist/schemas.d.ts +245 -0
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +58 -0
- package/dist/schemas.js.map +1 -1
- package/dist/thumbnail-schemas.d.ts +38 -28
- package/dist/thumbnail-schemas.d.ts.map +1 -1
- package/dist/tools/ask-brain.js +1 -1
- package/dist/tools/ask-brain.js.map +1 -1
- package/dist/tools/ask-brains.d.ts +66 -0
- package/dist/tools/ask-brains.d.ts.map +1 -0
- package/dist/tools/ask-brains.js +111 -0
- package/dist/tools/ask-brains.js.map +1 -0
- package/dist/tools/brain-info.js +1 -1
- package/dist/tools/brain-info.js.map +1 -1
- package/dist/tools/check-health.d.ts +75 -6
- package/dist/tools/check-health.d.ts.map +1 -1
- package/dist/tools/check-health.js +114 -6
- package/dist/tools/check-health.js.map +1 -1
- package/dist/tools/check-new-uploads.d.ts +55 -0
- package/dist/tools/check-new-uploads.d.ts.map +1 -0
- package/dist/tools/check-new-uploads.js +74 -0
- package/dist/tools/check-new-uploads.js.map +1 -0
- package/dist/tools/clips.d.ts +15 -0
- package/dist/tools/clips.d.ts.map +1 -1
- package/dist/tools/clips.js +5 -5
- package/dist/tools/clips.js.map +1 -1
- package/dist/tools/detect-scenes.d.ts +38 -0
- package/dist/tools/detect-scenes.d.ts.map +1 -0
- package/dist/tools/detect-scenes.js +66 -0
- package/dist/tools/detect-scenes.js.map +1 -0
- package/dist/tools/export-data.d.ts +85 -0
- package/dist/tools/export-data.d.ts.map +1 -0
- package/dist/tools/export-data.js +99 -0
- package/dist/tools/export-data.js.map +1 -0
- package/dist/tools/fetch-channel-thumbnails.d.ts +8 -8
- package/dist/tools/fetch-videos.d.ts +15 -4
- package/dist/tools/fetch-videos.d.ts.map +1 -1
- package/dist/tools/fetch-videos.js +63 -22
- package/dist/tools/fetch-videos.js.map +1 -1
- package/dist/tools/frames-subtitles.d.ts.map +1 -1
- package/dist/tools/frames-subtitles.js +16 -8
- package/dist/tools/frames-subtitles.js.map +1 -1
- package/dist/tools/frames.d.ts +73 -0
- package/dist/tools/frames.d.ts.map +1 -0
- package/dist/tools/frames.js +136 -0
- package/dist/tools/frames.js.map +1 -0
- package/dist/tools/get-channel-info.d.ts +2 -0
- package/dist/tools/get-channel-info.d.ts.map +1 -1
- package/dist/tools/get-channel-info.js +4 -0
- package/dist/tools/get-channel-info.js.map +1 -1
- package/dist/tools/get-comments.d.ts +238 -6
- package/dist/tools/get-comments.d.ts.map +1 -1
- package/dist/tools/get-comments.js +85 -21
- package/dist/tools/get-comments.js.map +1 -1
- package/dist/tools/get-coverage.d.ts +122 -0
- package/dist/tools/get-coverage.d.ts.map +1 -0
- package/dist/tools/get-coverage.js +105 -0
- package/dist/tools/get-coverage.js.map +1 -0
- package/dist/tools/get-sponsor-segments.d.ts +49 -0
- package/dist/tools/get-sponsor-segments.d.ts.map +1 -0
- package/dist/tools/get-sponsor-segments.js +61 -0
- package/dist/tools/get-sponsor-segments.js.map +1 -0
- package/dist/tools/get-thumbnail.d.ts.map +1 -1
- package/dist/tools/get-thumbnail.js +2 -1
- package/dist/tools/get-thumbnail.js.map +1 -1
- package/dist/tools/get-transcript.d.ts +1 -0
- package/dist/tools/get-transcript.d.ts.map +1 -1
- package/dist/tools/get-transcript.js +11 -4
- package/dist/tools/get-transcript.js.map +1 -1
- package/dist/tools/get-video-details.d.ts +118 -0
- package/dist/tools/get-video-details.d.ts.map +1 -0
- package/dist/tools/get-video-details.js +147 -0
- package/dist/tools/get-video-details.js.map +1 -0
- package/dist/tools/gif-concat.d.ts +49 -0
- package/dist/tools/gif-concat.d.ts.map +1 -0
- package/dist/tools/gif-concat.js +96 -0
- package/dist/tools/gif-concat.js.map +1 -0
- package/dist/tools/harvest-channel.d.ts +92 -0
- package/dist/tools/harvest-channel.d.ts.map +1 -0
- package/dist/tools/harvest-channel.js +62 -0
- package/dist/tools/harvest-channel.js.map +1 -0
- package/dist/tools/harvest-comments.d.ts +81 -0
- package/dist/tools/harvest-comments.d.ts.map +1 -0
- package/dist/tools/harvest-comments.js +115 -0
- package/dist/tools/harvest-comments.js.map +1 -0
- package/dist/tools/harvest-details.d.ts +84 -0
- package/dist/tools/harvest-details.d.ts.map +1 -0
- package/dist/tools/harvest-details.js +71 -0
- package/dist/tools/harvest-details.js.map +1 -0
- package/dist/tools/library.js +1 -1
- package/dist/tools/library.js.map +1 -1
- package/dist/tools/list-captions.d.ts +17 -0
- package/dist/tools/list-captions.d.ts.map +1 -0
- package/dist/tools/list-captions.js +60 -0
- package/dist/tools/list-captions.js.map +1 -0
- package/dist/tools/list-channel-playlists.d.ts +30 -0
- package/dist/tools/list-channel-playlists.d.ts.map +1 -0
- package/dist/tools/list-channel-playlists.js +44 -0
- package/dist/tools/list-channel-playlists.js.map +1 -0
- package/dist/tools/list-channel-thumbnails.d.ts +10 -10
- package/dist/tools/list-channel-thumbnails.js +1 -1
- package/dist/tools/list-channel-thumbnails.js.map +1 -1
- package/dist/tools/list-library.js +1 -1
- package/dist/tools/list-library.js.map +1 -1
- package/dist/tools/prune-harvest.d.ts +59 -0
- package/dist/tools/prune-harvest.d.ts.map +1 -0
- package/dist/tools/prune-harvest.js +108 -0
- package/dist/tools/prune-harvest.js.map +1 -0
- package/dist/tools/query-comments.d.ts +125 -0
- package/dist/tools/query-comments.d.ts.map +1 -0
- package/dist/tools/query-comments.js +106 -0
- package/dist/tools/query-comments.js.map +1 -0
- package/dist/tools/query-videos.d.ts +118 -0
- package/dist/tools/query-videos.d.ts.map +1 -0
- package/dist/tools/query-videos.js +97 -0
- package/dist/tools/query-videos.js.map +1 -0
- package/dist/tools/repair-store.d.ts +34 -0
- package/dist/tools/repair-store.d.ts.map +1 -0
- package/dist/tools/repair-store.js +58 -0
- package/dist/tools/repair-store.js.map +1 -0
- package/dist/tools/search-channels.d.ts +6 -0
- package/dist/tools/search-channels.d.ts.map +1 -1
- package/dist/tools/search-channels.js +5 -1
- package/dist/tools/search-channels.js.map +1 -1
- package/dist/tools/search-videos.js +1 -1
- package/dist/tools/search-videos.js.map +1 -1
- package/dist/tools/transcribe-video.d.ts +28 -0
- package/dist/tools/transcribe-video.d.ts.map +1 -0
- package/dist/tools/transcribe-video.js +52 -0
- package/dist/tools/transcribe-video.js.map +1 -0
- package/dist/utils/brain-fusion.d.ts +24 -0
- package/dist/utils/brain-fusion.d.ts.map +1 -0
- package/dist/utils/brain-fusion.js +18 -0
- package/dist/utils/brain-fusion.js.map +1 -0
- package/dist/utils/channel-playlists.d.ts +21 -0
- package/dist/utils/channel-playlists.d.ts.map +1 -0
- package/dist/utils/channel-playlists.js +56 -0
- package/dist/utils/channel-playlists.js.map +1 -0
- package/dist/utils/clips.d.ts.map +1 -1
- package/dist/utils/clips.js +3 -17
- package/dist/utils/clips.js.map +1 -1
- package/dist/utils/comment-store.d.ts +71 -0
- package/dist/utils/comment-store.d.ts.map +1 -0
- package/dist/utils/comment-store.js +129 -0
- package/dist/utils/comment-store.js.map +1 -0
- package/dist/utils/comment-threads.d.ts +51 -0
- package/dist/utils/comment-threads.d.ts.map +1 -0
- package/dist/utils/comment-threads.js +76 -0
- package/dist/utils/comment-threads.js.map +1 -0
- package/dist/utils/concat.d.ts +19 -0
- package/dist/utils/concat.d.ts.map +1 -0
- package/dist/utils/concat.js +135 -0
- package/dist/utils/concat.js.map +1 -0
- package/dist/utils/contact-sheet.d.ts +28 -0
- package/dist/utils/contact-sheet.d.ts.map +1 -0
- package/dist/utils/contact-sheet.js +65 -0
- package/dist/utils/contact-sheet.js.map +1 -0
- package/dist/utils/coverage-text.d.ts +3 -0
- package/dist/utils/coverage-text.d.ts.map +1 -0
- package/dist/utils/coverage-text.js +62 -0
- package/dist/utils/coverage-text.js.map +1 -0
- package/dist/utils/coverage.d.ts +57 -0
- package/dist/utils/coverage.d.ts.map +1 -0
- package/dist/utils/coverage.js +142 -0
- package/dist/utils/coverage.js.map +1 -0
- package/dist/utils/detail-batch.d.ts +25 -0
- package/dist/utils/detail-batch.d.ts.map +1 -0
- package/dist/utils/detail-batch.js +58 -0
- package/dist/utils/detail-batch.js.map +1 -0
- package/dist/utils/errors.d.ts +1 -1
- package/dist/utils/errors.d.ts.map +1 -1
- package/dist/utils/errors.js +19 -1
- package/dist/utils/errors.js.map +1 -1
- package/dist/utils/export-data.d.ts +41 -0
- package/dist/utils/export-data.d.ts.map +1 -0
- package/dist/utils/export-data.js +114 -0
- package/dist/utils/export-data.js.map +1 -0
- package/dist/utils/export-format.d.ts +23 -0
- package/dist/utils/export-format.d.ts.map +1 -0
- package/dist/utils/export-format.js +59 -0
- package/dist/utils/export-format.js.map +1 -0
- package/dist/utils/export-load.d.ts +28 -0
- package/dist/utils/export-load.d.ts.map +1 -0
- package/dist/utils/export-load.js +70 -0
- package/dist/utils/export-load.js.map +1 -0
- package/dist/utils/export-rows.d.ts +35 -0
- package/dist/utils/export-rows.d.ts.map +1 -0
- package/dist/utils/export-rows.js +69 -0
- package/dist/utils/export-rows.js.map +1 -0
- package/dist/utils/ffmpeg.d.ts +26 -0
- package/dist/utils/ffmpeg.d.ts.map +1 -0
- package/dist/utils/ffmpeg.js +28 -0
- package/dist/utils/ffmpeg.js.map +1 -0
- package/dist/utils/format.d.ts +23 -1
- package/dist/utils/format.d.ts.map +1 -1
- package/dist/utils/format.js +5 -1
- package/dist/utils/format.js.map +1 -1
- package/dist/utils/frame-plan.d.ts +24 -0
- package/dist/utils/frame-plan.d.ts.map +1 -0
- package/dist/utils/frame-plan.js +58 -0
- package/dist/utils/frame-plan.js.map +1 -0
- package/dist/utils/frames.d.ts +32 -0
- package/dist/utils/frames.d.ts.map +1 -0
- package/dist/utils/frames.js +60 -0
- package/dist/utils/frames.js.map +1 -0
- package/dist/utils/gif.d.ts +31 -0
- package/dist/utils/gif.d.ts.map +1 -0
- package/dist/utils/gif.js +77 -0
- package/dist/utils/gif.js.map +1 -0
- package/dist/utils/harvest-catalog.d.ts +54 -0
- package/dist/utils/harvest-catalog.d.ts.map +1 -0
- package/dist/utils/harvest-catalog.js +173 -0
- package/dist/utils/harvest-catalog.js.map +1 -0
- package/dist/utils/harvest-details.d.ts +30 -0
- package/dist/utils/harvest-details.d.ts.map +1 -0
- package/dist/utils/harvest-details.js +98 -0
- package/dist/utils/harvest-details.js.map +1 -0
- package/dist/utils/harvest-lock.d.ts +3 -0
- package/dist/utils/harvest-lock.d.ts.map +1 -0
- package/dist/utils/harvest-lock.js +26 -0
- package/dist/utils/harvest-lock.js.map +1 -0
- package/dist/utils/harvest-receipts.d.ts +14 -0
- package/dist/utils/harvest-receipts.d.ts.map +1 -0
- package/dist/utils/harvest-receipts.js +110 -0
- package/dist/utils/harvest-receipts.js.map +1 -0
- package/dist/utils/listing-cursor.d.ts +31 -0
- package/dist/utils/listing-cursor.d.ts.map +1 -0
- package/dist/utils/listing-cursor.js +39 -0
- package/dist/utils/listing-cursor.js.map +1 -0
- package/dist/utils/local-transcription.d.ts +14 -0
- package/dist/utils/local-transcription.d.ts.map +1 -0
- package/dist/utils/local-transcription.js +85 -0
- package/dist/utils/local-transcription.js.map +1 -0
- package/dist/utils/moment.d.ts +24 -0
- package/dist/utils/moment.d.ts.map +1 -0
- package/dist/utils/moment.js +22 -0
- package/dist/utils/moment.js.map +1 -0
- package/dist/utils/most-replayed.d.ts +15 -0
- package/dist/utils/most-replayed.d.ts.map +1 -0
- package/dist/utils/most-replayed.js +25 -0
- package/dist/utils/most-replayed.js.map +1 -0
- package/dist/utils/pot-preflight.d.ts +53 -0
- package/dist/utils/pot-preflight.d.ts.map +1 -0
- package/dist/utils/pot-preflight.js +146 -0
- package/dist/utils/pot-preflight.js.map +1 -0
- package/dist/utils/scenes.d.ts +39 -0
- package/dist/utils/scenes.d.ts.map +1 -0
- package/dist/utils/scenes.js +83 -0
- package/dist/utils/scenes.js.map +1 -0
- package/dist/utils/seconds.d.ts +5 -0
- package/dist/utils/seconds.d.ts.map +1 -0
- package/dist/utils/seconds.js +7 -0
- package/dist/utils/seconds.js.map +1 -0
- package/dist/utils/sleep.d.ts +9 -0
- package/dist/utils/sleep.d.ts.map +1 -0
- package/dist/utils/sleep.js +25 -0
- package/dist/utils/sleep.js.map +1 -0
- package/dist/utils/sponsorblock.d.ts +24 -0
- package/dist/utils/sponsorblock.d.ts.map +1 -0
- package/dist/utils/sponsorblock.js +153 -0
- package/dist/utils/sponsorblock.js.map +1 -0
- package/dist/utils/store-health.d.ts +37 -0
- package/dist/utils/store-health.d.ts.map +1 -0
- package/dist/utils/store-health.js +92 -0
- package/dist/utils/store-health.js.map +1 -0
- package/dist/utils/store-migrations.d.ts +33 -0
- package/dist/utils/store-migrations.d.ts.map +1 -0
- package/dist/utils/store-migrations.js +41 -0
- package/dist/utils/store-migrations.js.map +1 -0
- package/dist/utils/store-paths.d.ts +33 -0
- package/dist/utils/store-paths.d.ts.map +1 -0
- package/dist/utils/store-paths.js +48 -0
- package/dist/utils/store-paths.js.map +1 -0
- package/dist/utils/store-rows.d.ts +22 -0
- package/dist/utils/store-rows.d.ts.map +1 -0
- package/dist/utils/store-rows.js +62 -0
- package/dist/utils/store-rows.js.map +1 -0
- package/dist/utils/store-schema.d.ts +16 -0
- package/dist/utils/store-schema.d.ts.map +1 -0
- package/dist/utils/store-schema.js +168 -0
- package/dist/utils/store-schema.js.map +1 -0
- package/dist/utils/store.d.ts +34 -0
- package/dist/utils/store.d.ts.map +1 -0
- package/dist/utils/store.js +157 -0
- package/dist/utils/store.js.map +1 -0
- package/dist/utils/transcript-cache.d.ts +5 -1
- package/dist/utils/transcript-cache.d.ts.map +1 -1
- package/dist/utils/transcript-cache.js +31 -50
- package/dist/utils/transcript-cache.js.map +1 -1
- package/dist/utils/transcript-files.d.ts +24 -0
- package/dist/utils/transcript-files.d.ts.map +1 -0
- package/dist/utils/transcript-files.js +61 -0
- package/dist/utils/transcript-files.js.map +1 -0
- package/dist/utils/transcript.d.ts +3 -0
- package/dist/utils/transcript.d.ts.map +1 -1
- package/dist/utils/transcript.js +2 -0
- package/dist/utils/transcript.js.map +1 -1
- package/dist/utils/upload-check.d.ts +38 -0
- package/dist/utils/upload-check.d.ts.map +1 -0
- package/dist/utils/upload-check.js +93 -0
- package/dist/utils/upload-check.js.map +1 -0
- package/dist/utils/validate.d.ts +2 -0
- package/dist/utils/validate.d.ts.map +1 -1
- package/dist/utils/validate.js +4 -0
- package/dist/utils/validate.js.map +1 -1
- package/dist/utils/video-detail-store.d.ts +47 -0
- package/dist/utils/video-detail-store.d.ts.map +1 -0
- package/dist/utils/video-detail-store.js +89 -0
- package/dist/utils/video-detail-store.js.map +1 -0
- package/dist/utils/video-stats-cache.d.ts +16 -0
- package/dist/utils/video-stats-cache.d.ts.map +1 -0
- package/dist/utils/video-stats-cache.js +81 -0
- package/dist/utils/video-stats-cache.js.map +1 -0
- package/dist/utils/video-stats-schema.d.ts +187 -0
- package/dist/utils/video-stats-schema.d.ts.map +1 -0
- package/dist/utils/video-stats-schema.js +50 -0
- package/dist/utils/video-stats-schema.js.map +1 -0
- package/dist/utils/video-stats-storage.d.ts +16 -0
- package/dist/utils/video-stats-storage.d.ts.map +1 -0
- package/dist/utils/video-stats-storage.js +37 -0
- package/dist/utils/video-stats-storage.js.map +1 -0
- package/dist/utils/video-stats.d.ts +82 -0
- package/dist/utils/video-stats.d.ts.map +1 -0
- package/dist/utils/video-stats.js +108 -0
- package/dist/utils/video-stats.js.map +1 -0
- package/dist/utils/video-store.d.ts +69 -0
- package/dist/utils/video-store.d.ts.map +1 -0
- package/dist/utils/video-store.js +77 -0
- package/dist/utils/video-store.js.map +1 -0
- package/dist/utils/video-stream.d.ts +21 -0
- package/dist/utils/video-stream.d.ts.map +1 -0
- package/dist/utils/video-stream.js +36 -0
- package/dist/utils/video-stream.js.map +1 -0
- package/dist/utils/whisper.d.ts +39 -0
- package/dist/utils/whisper.d.ts.map +1 -0
- package/dist/utils/whisper.js +137 -0
- package/dist/utils/whisper.js.map +1 -0
- package/dist/utils/youtube-channel.d.ts +13 -2
- package/dist/utils/youtube-channel.d.ts.map +1 -1
- package/dist/utils/youtube-channel.js +39 -3
- package/dist/utils/youtube-channel.js.map +1 -1
- package/dist/utils/youtube-schemas.d.ts +104 -0
- package/dist/utils/youtube-schemas.d.ts.map +1 -1
- package/dist/utils/youtube-schemas.js +32 -0
- package/dist/utils/youtube-schemas.js.map +1 -1
- package/dist/utils/youtube-video.d.ts +22 -1
- package/dist/utils/youtube-video.d.ts.map +1 -1
- package/dist/utils/youtube-video.js +40 -28
- package/dist/utils/youtube-video.js.map +1 -1
- package/dist/utils/ytdlp-limiter.d.ts +3 -0
- package/dist/utils/ytdlp-limiter.d.ts.map +1 -1
- package/dist/utils/ytdlp-limiter.js +19 -3
- package/dist/utils/ytdlp-limiter.js.map +1 -1
- package/dist/utils/ytdlp-pacer.d.ts +64 -0
- package/dist/utils/ytdlp-pacer.d.ts.map +1 -0
- package/dist/utils/ytdlp-pacer.js +165 -0
- package/dist/utils/ytdlp-pacer.js.map +1 -0
- package/dist/utils/ytdlp-stream.d.ts +32 -0
- package/dist/utils/ytdlp-stream.d.ts.map +1 -0
- package/dist/utils/ytdlp-stream.js +61 -0
- package/dist/utils/ytdlp-stream.js.map +1 -0
- package/dist/utils/ytdlp.d.ts +10 -0
- package/dist/utils/ytdlp.d.ts.map +1 -1
- package/dist/utils/ytdlp.js +36 -25
- package/dist/utils/ytdlp.js.map +1 -1
- package/package.json +10 -4
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Finding where a video cuts.
|
|
3
|
+
*
|
|
4
|
+
* ffmpeg scores every frame by how different it is from the one before; frames
|
|
5
|
+
* scoring above the threshold are cuts. Detection has to decode every frame of the
|
|
6
|
+
* window, so it reads the lightest stream and only a bounded window at a time.
|
|
7
|
+
*/
|
|
8
|
+
/** A scan decodes every frame; this keeps one call to a few minutes of work. */
|
|
9
|
+
export declare const MAX_SCENE_WINDOW_SECONDS = 900;
|
|
10
|
+
export declare const MAX_SCENE_CUTS = 200;
|
|
11
|
+
export declare const DEFAULT_SCENE_THRESHOLD = 0.4;
|
|
12
|
+
export interface ScanWindowRequest {
|
|
13
|
+
start?: number;
|
|
14
|
+
end?: number;
|
|
15
|
+
}
|
|
16
|
+
export interface SceneScanRequest extends ScanWindowRequest {
|
|
17
|
+
/** 0 to 1: how different a frame must be from the last to count as a cut. */
|
|
18
|
+
threshold: number;
|
|
19
|
+
}
|
|
20
|
+
export interface SceneScan {
|
|
21
|
+
videoId: string;
|
|
22
|
+
title: string;
|
|
23
|
+
duration: number;
|
|
24
|
+
start: number;
|
|
25
|
+
end: number;
|
|
26
|
+
threshold: number;
|
|
27
|
+
/** Absolute video time of each cut, ascending. */
|
|
28
|
+
cuts: number[];
|
|
29
|
+
/** True when more cuts were found than are returned. */
|
|
30
|
+
truncated: boolean;
|
|
31
|
+
}
|
|
32
|
+
/** Frames ffmpeg selected, as the times its showinfo filter logged them at. */
|
|
33
|
+
export declare function parseSceneCuts(stderr: string, offset: number): number[];
|
|
34
|
+
export declare function planScanWindow(request: ScanWindowRequest, duration: number): {
|
|
35
|
+
start: number;
|
|
36
|
+
end: number;
|
|
37
|
+
};
|
|
38
|
+
export declare function detectScenes(video: string, request: SceneScanRequest): Promise<SceneScan>;
|
|
39
|
+
//# sourceMappingURL=scenes.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scenes.d.ts","sourceRoot":"","sources":["../../src/utils/scenes.ts"],"names":[],"mappings":"AAWA;;;;;;GAMG;AAEH,gFAAgF;AAChF,eAAO,MAAM,wBAAwB,MAAM,CAAC;AAC5C,eAAO,MAAM,cAAc,MAAM,CAAC;AAClC,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAG3C,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,gBAAiB,SAAQ,iBAAiB;IACzD,6EAA6E;IAC7E,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,SAAS;IACxB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;IAClB,kDAAkD;IAClD,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,wDAAwD;IACxD,SAAS,EAAE,OAAO,CAAC;CACpB;AAED,+EAA+E;AAC/E,wBAAgB,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAUvE;AAED,wBAAgB,cAAc,CAC5B,OAAO,EAAE,iBAAiB,EAC1B,QAAQ,EAAE,MAAM,GACf;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CA0BhC;AAED,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,SAAS,CAAC,CA2C/F"}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { YouTubeError } from './errors.js';
|
|
2
|
+
import { runFfmpeg } from './ffmpeg.js';
|
|
3
|
+
import { roundToMillisecond } from './seconds.js';
|
|
4
|
+
import { requireFfmpeg } from './preflight.js';
|
|
5
|
+
import { FFMPEG_NETWORK_INPUT_ARGS, resolveStreamUrl, SCAN_STREAM_SELECTOR, } from './video-stream.js';
|
|
6
|
+
import { getVideoInfo } from './youtube.js';
|
|
7
|
+
/**
|
|
8
|
+
* Finding where a video cuts.
|
|
9
|
+
*
|
|
10
|
+
* ffmpeg scores every frame by how different it is from the one before; frames
|
|
11
|
+
* scoring above the threshold are cuts. Detection has to decode every frame of the
|
|
12
|
+
* window, so it reads the lightest stream and only a bounded window at a time.
|
|
13
|
+
*/
|
|
14
|
+
/** A scan decodes every frame; this keeps one call to a few minutes of work. */
|
|
15
|
+
export const MAX_SCENE_WINDOW_SECONDS = 900;
|
|
16
|
+
export const MAX_SCENE_CUTS = 200;
|
|
17
|
+
export const DEFAULT_SCENE_THRESHOLD = 0.4;
|
|
18
|
+
const SCAN_TIMEOUT_MS = 300_000;
|
|
19
|
+
/** Frames ffmpeg selected, as the times its showinfo filter logged them at. */
|
|
20
|
+
export function parseSceneCuts(stderr, offset) {
|
|
21
|
+
const times = stderr
|
|
22
|
+
.split('\n')
|
|
23
|
+
.filter((line) => line.includes('Parsed_showinfo'))
|
|
24
|
+
.flatMap((line) => {
|
|
25
|
+
const match = /pts_time:(\d+(?:\.\d+)?)/.exec(line);
|
|
26
|
+
return match?.[1] === undefined ? [] : [roundToMillisecond(offset + Number(match[1]))];
|
|
27
|
+
});
|
|
28
|
+
return [...new Set(times)].sort((a, b) => a - b);
|
|
29
|
+
}
|
|
30
|
+
export function planScanWindow(request, duration) {
|
|
31
|
+
const { start = 0 } = request;
|
|
32
|
+
if (duration > 0 && start >= duration) {
|
|
33
|
+
throw new YouTubeError('INVALID_INPUT', `${Math.round(start)}s is past the end of this ${duration}s video.`, { nextStep: 'Pick a start time inside the video.' });
|
|
34
|
+
}
|
|
35
|
+
const requestedEnd = request.end ?? start + MAX_SCENE_WINDOW_SECONDS;
|
|
36
|
+
if (requestedEnd <= start) {
|
|
37
|
+
throw new YouTubeError('INVALID_INPUT', 'end must be later than start.');
|
|
38
|
+
}
|
|
39
|
+
const end = duration > 0 ? Math.min(requestedEnd, duration) : requestedEnd;
|
|
40
|
+
if (end - start > MAX_SCENE_WINDOW_SECONDS) {
|
|
41
|
+
throw new YouTubeError('INVALID_INPUT', `Scene detection reads at most ${MAX_SCENE_WINDOW_SECONDS}s per call.`, { nextStep: 'Scan the video in windows, moving start forward each time.' });
|
|
42
|
+
}
|
|
43
|
+
return { start, end };
|
|
44
|
+
}
|
|
45
|
+
export async function detectScenes(video, request) {
|
|
46
|
+
await requireFfmpeg('Scene detection');
|
|
47
|
+
const info = await getVideoInfo(video);
|
|
48
|
+
const { start, end } = planScanWindow(request, info.duration);
|
|
49
|
+
const { threshold } = request;
|
|
50
|
+
const mediaUrl = await resolveStreamUrl(info.id, SCAN_STREAM_SELECTOR, 'Scene detection');
|
|
51
|
+
const { stderr } = await runFfmpeg([
|
|
52
|
+
...FFMPEG_NETWORK_INPUT_ARGS,
|
|
53
|
+
// Both before -i: seek there, then read only the window.
|
|
54
|
+
'-ss',
|
|
55
|
+
String(start),
|
|
56
|
+
'-t',
|
|
57
|
+
String(end - start),
|
|
58
|
+
'-i',
|
|
59
|
+
mediaUrl,
|
|
60
|
+
'-an',
|
|
61
|
+
'-vf',
|
|
62
|
+
`select=gt(scene\\,${String(threshold)}),showinfo`,
|
|
63
|
+
'-f',
|
|
64
|
+
'null',
|
|
65
|
+
'-',
|
|
66
|
+
], {
|
|
67
|
+
failure: 'ffmpeg could not scan this video for scene cuts.',
|
|
68
|
+
nextStep: 'Try a shorter window, or check that ffmpeg works with network input.',
|
|
69
|
+
timeoutMs: SCAN_TIMEOUT_MS,
|
|
70
|
+
});
|
|
71
|
+
const cuts = parseSceneCuts(stderr, start);
|
|
72
|
+
return {
|
|
73
|
+
videoId: info.id,
|
|
74
|
+
title: info.title,
|
|
75
|
+
duration: info.duration,
|
|
76
|
+
start,
|
|
77
|
+
end,
|
|
78
|
+
threshold,
|
|
79
|
+
cuts: cuts.slice(0, MAX_SCENE_CUTS),
|
|
80
|
+
truncated: cuts.length > MAX_SCENE_CUTS,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=scenes.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scenes.js","sourceRoot":"","sources":["../../src/utils/scenes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EACL,yBAAyB,EACzB,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAE5C;;;;;;GAMG;AAEH,gFAAgF;AAChF,MAAM,CAAC,MAAM,wBAAwB,GAAG,GAAG,CAAC;AAC5C,MAAM,CAAC,MAAM,cAAc,GAAG,GAAG,CAAC;AAClC,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAC3C,MAAM,eAAe,GAAG,OAAO,CAAC;AAyBhC,+EAA+E;AAC/E,MAAM,UAAU,cAAc,CAAC,MAAc,EAAE,MAAc;IAC3D,MAAM,KAAK,GAAG,MAAM;SACjB,KAAK,CAAC,IAAI,CAAC;SACX,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;SAClD,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE;QAChB,MAAM,KAAK,GAAG,0BAA0B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpD,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;IAEL,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,OAA0B,EAC1B,QAAgB;IAEhB,MAAM,EAAE,KAAK,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAE9B,IAAI,QAAQ,GAAG,CAAC,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;QACtC,MAAM,IAAI,YAAY,CACpB,eAAe,EACf,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,6BAA6B,QAAQ,UAAU,EACnE,EAAE,QAAQ,EAAE,qCAAqC,EAAE,CACpD,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,IAAI,KAAK,GAAG,wBAAwB,CAAC;IACrE,IAAI,YAAY,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,IAAI,YAAY,CAAC,eAAe,EAAE,+BAA+B,CAAC,CAAC;IAC3E,CAAC;IAED,MAAM,GAAG,GAAG,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC;IAC3E,IAAI,GAAG,GAAG,KAAK,GAAG,wBAAwB,EAAE,CAAC;QAC3C,MAAM,IAAI,YAAY,CACpB,eAAe,EACf,iCAAiC,wBAAwB,aAAa,EACtE,EAAE,QAAQ,EAAE,4DAA4D,EAAE,CAC3E,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAa,EAAE,OAAyB;IACzE,MAAM,aAAa,CAAC,iBAAiB,CAAC,CAAC;IACvC,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;IACvC,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC9D,MAAM,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;IAE9B,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,IAAI,CAAC,EAAE,EAAE,oBAAoB,EAAE,iBAAiB,CAAC,CAAC;IAE1F,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAChC;QACE,GAAG,yBAAyB;QAC5B,yDAAyD;QACzD,KAAK;QACL,MAAM,CAAC,KAAK,CAAC;QACb,IAAI;QACJ,MAAM,CAAC,GAAG,GAAG,KAAK,CAAC;QACnB,IAAI;QACJ,QAAQ;QACR,KAAK;QACL,KAAK;QACL,qBAAqB,MAAM,CAAC,SAAS,CAAC,YAAY;QAClD,IAAI;QACJ,MAAM;QACN,GAAG;KACJ,EACD;QACE,OAAO,EAAE,kDAAkD;QAC3D,QAAQ,EAAE,sEAAsE;QAChF,SAAS,EAAE,eAAe;KAC3B,CACF,CAAC;IAEF,MAAM,IAAI,GAAG,cAAc,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAC3C,OAAO;QACL,OAAO,EAAE,IAAI,CAAC,EAAE;QAChB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,KAAK;QACL,GAAG;QACH,SAAS;QACT,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC;QACnC,SAAS,EAAE,IAAI,CAAC,MAAM,GAAG,cAAc;KACxC,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/** Time in the units the tools speak (seconds) and the units ffmpeg and whisper.cpp write (milliseconds). */
|
|
2
|
+
export declare const MILLISECONDS_PER_SECOND = 1000;
|
|
3
|
+
/** Seconds rounded to the nearest millisecond, which is all the precision any source here has. */
|
|
4
|
+
export declare function roundToMillisecond(seconds: number): number;
|
|
5
|
+
//# sourceMappingURL=seconds.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"seconds.d.ts","sourceRoot":"","sources":["../../src/utils/seconds.ts"],"names":[],"mappings":"AAAA,6GAA6G;AAE7G,eAAO,MAAM,uBAAuB,OAAO,CAAC;AAE5C,kGAAkG;AAClG,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE1D"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/** Time in the units the tools speak (seconds) and the units ffmpeg and whisper.cpp write (milliseconds). */
|
|
2
|
+
export const MILLISECONDS_PER_SECOND = 1000;
|
|
3
|
+
/** Seconds rounded to the nearest millisecond, which is all the precision any source here has. */
|
|
4
|
+
export function roundToMillisecond(seconds) {
|
|
5
|
+
return Math.round(seconds * MILLISECONDS_PER_SECOND) / MILLISECONDS_PER_SECOND;
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=seconds.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"seconds.js","sourceRoot":"","sources":["../../src/utils/seconds.ts"],"names":[],"mappings":"AAAA,6GAA6G;AAE7G,MAAM,CAAC,MAAM,uBAAuB,GAAG,IAAI,CAAC;AAE5C,kGAAkG;AAClG,MAAM,UAAU,kBAAkB,CAAC,OAAe;IAChD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,uBAAuB,CAAC,GAAG,uBAAuB,CAAC;AACjF,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A cancellable pause.
|
|
3
|
+
*
|
|
4
|
+
* Shared by the retry loop and the pacer so a cooling-down server still
|
|
5
|
+
* answers a client that gives up: an uncancellable sleep is how a cooldown
|
|
6
|
+
* turns into a wedge.
|
|
7
|
+
*/
|
|
8
|
+
export declare function sleep(ms: number, signal?: AbortSignal): Promise<void>;
|
|
9
|
+
//# sourceMappingURL=sleep.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sleep.d.ts","sourceRoot":"","sources":["../../src/utils/sleep.ts"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,wBAAsB,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB3E"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { YouTubeError } from './errors.js';
|
|
2
|
+
/**
|
|
3
|
+
* A cancellable pause.
|
|
4
|
+
*
|
|
5
|
+
* Shared by the retry loop and the pacer so a cooling-down server still
|
|
6
|
+
* answers a client that gives up: an uncancellable sleep is how a cooldown
|
|
7
|
+
* turns into a wedge.
|
|
8
|
+
*/
|
|
9
|
+
export async function sleep(ms, signal) {
|
|
10
|
+
if (signal?.aborted === true) {
|
|
11
|
+
throw new YouTubeError('CANCELLED', 'The request was cancelled.');
|
|
12
|
+
}
|
|
13
|
+
await new Promise((resolve, reject) => {
|
|
14
|
+
const timer = setTimeout(() => {
|
|
15
|
+
signal?.removeEventListener('abort', onAbort);
|
|
16
|
+
resolve();
|
|
17
|
+
}, ms);
|
|
18
|
+
const onAbort = () => {
|
|
19
|
+
clearTimeout(timer);
|
|
20
|
+
reject(new YouTubeError('CANCELLED', 'The request was cancelled.'));
|
|
21
|
+
};
|
|
22
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=sleep.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sleep.js","sourceRoot":"","sources":["../../src/utils/sleep.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,EAAU,EAAE,MAAoB;IAC1D,IAAI,MAAM,EAAE,OAAO,KAAK,IAAI,EAAE,CAAC;QAC7B,MAAM,IAAI,YAAY,CAAC,WAAW,EAAE,4BAA4B,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC9C,OAAO,EAAE,CAAC;QACZ,CAAC,EAAE,EAAE,CAAC,CAAC;QACP,MAAM,OAAO,GAAG,GAAS,EAAE;YACzB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,IAAI,YAAY,CAAC,WAAW,EAAE,4BAA4B,CAAC,CAAC,CAAC;QACtE,CAAC,CAAC;QACF,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** SponsorBlock's documented categories that mark something a viewer may want to skip. */
|
|
2
|
+
export declare const SPONSOR_CATEGORIES: readonly ["sponsor", "selfpromo", "interaction", "intro", "outro", "preview", "music_offtopic", "filler"];
|
|
3
|
+
export type SponsorCategory = (typeof SPONSOR_CATEGORIES)[number];
|
|
4
|
+
export interface SponsorSegment {
|
|
5
|
+
category: string;
|
|
6
|
+
startSeconds: number;
|
|
7
|
+
endSeconds: number;
|
|
8
|
+
durationSeconds: number;
|
|
9
|
+
/** Community votes: higher is more agreed upon. */
|
|
10
|
+
votes: number;
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* The length covered by the segments, with overlaps counted once.
|
|
14
|
+
*
|
|
15
|
+
* SponsorBlock categories overlap (a sponsor read inside a self-promotion) and several
|
|
16
|
+
* submissions can mark the same range, so a plain sum of durations is more than the
|
|
17
|
+
* video can hold.
|
|
18
|
+
*/
|
|
19
|
+
export declare function coveredSeconds(segments: readonly SponsorSegment[]): number;
|
|
20
|
+
export declare function hashPrefix(videoId: string): string;
|
|
21
|
+
export declare function fetchSponsorSegments(videoId: string, categories?: readonly string[], options?: {
|
|
22
|
+
timeoutMs?: number;
|
|
23
|
+
}): Promise<SponsorSegment[]>;
|
|
24
|
+
//# sourceMappingURL=sponsorblock.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sponsorblock.d.ts","sourceRoot":"","sources":["../../src/utils/sponsorblock.ts"],"names":[],"mappings":"AAuBA,0FAA0F;AAC1F,eAAO,MAAM,kBAAkB,2GASrB,CAAC;AAEX,MAAM,MAAM,eAAe,GAAG,CAAC,OAAO,kBAAkB,CAAC,CAAC,MAAM,CAAC,CAAC;AAElE,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,mDAAmD;IACnD,KAAK,EAAE,MAAM,CAAC;CACf;AAeD;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,GAAG,MAAM,CAa1E;AAED,wBAAgB,UAAU,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAElD;AA8DD,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,MAAM,EACf,UAAU,GAAE,SAAS,MAAM,EAAuB,EAClD,OAAO,GAAE;IAAE,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GACnC,OAAO,CAAC,cAAc,EAAE,CAAC,CA6C3B"}
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { currentContext } from './context.js';
|
|
4
|
+
import { YouTubeError } from './errors.js';
|
|
5
|
+
import { assertVideoId } from './validate.js';
|
|
6
|
+
/**
|
|
7
|
+
* The stretches of a video the community has marked as sponsor reads, intros, outros
|
|
8
|
+
* and the like, from SponsorBlock (https://sponsor.ajay.app), the database the whole
|
|
9
|
+
* ecosystem uses.
|
|
10
|
+
*
|
|
11
|
+
* Its API is queried by the first four hex digits of the video id's SHA-256, never by
|
|
12
|
+
* the id, so the service cannot tell which of the several videos sharing a prefix was
|
|
13
|
+
* asked about. That is the privacy design SponsorBlock documents, and the only way this
|
|
14
|
+
* module asks.
|
|
15
|
+
*/
|
|
16
|
+
const API = 'https://sponsor.ajay.app/api/skipSegments';
|
|
17
|
+
const HASH_PREFIX_LENGTH = 4;
|
|
18
|
+
const DEFAULT_TIMEOUT_MS = 15_000;
|
|
19
|
+
/** A hash prefix answers with every video sharing it; far more than this is not plausible. */
|
|
20
|
+
const MAX_RESPONSE_CHARS = 8_000_000;
|
|
21
|
+
/** SponsorBlock's documented categories that mark something a viewer may want to skip. */
|
|
22
|
+
export const SPONSOR_CATEGORIES = [
|
|
23
|
+
'sponsor',
|
|
24
|
+
'selfpromo',
|
|
25
|
+
'interaction',
|
|
26
|
+
'intro',
|
|
27
|
+
'outro',
|
|
28
|
+
'preview',
|
|
29
|
+
'music_offtopic',
|
|
30
|
+
'filler',
|
|
31
|
+
];
|
|
32
|
+
const responseSchema = z.array(z.object({
|
|
33
|
+
videoID: z.string(),
|
|
34
|
+
segments: z.array(z.object({
|
|
35
|
+
category: z.string(),
|
|
36
|
+
segment: z.tuple([z.number(), z.number()]),
|
|
37
|
+
votes: z.number().optional(),
|
|
38
|
+
})),
|
|
39
|
+
}));
|
|
40
|
+
/**
|
|
41
|
+
* The length covered by the segments, with overlaps counted once.
|
|
42
|
+
*
|
|
43
|
+
* SponsorBlock categories overlap (a sponsor read inside a self-promotion) and several
|
|
44
|
+
* submissions can mark the same range, so a plain sum of durations is more than the
|
|
45
|
+
* video can hold.
|
|
46
|
+
*/
|
|
47
|
+
export function coveredSeconds(segments) {
|
|
48
|
+
let covered = 0;
|
|
49
|
+
let reachedTo = Number.NEGATIVE_INFINITY;
|
|
50
|
+
for (const { startSeconds, endSeconds } of [...segments].sort((a, b) => a.startSeconds - b.startSeconds)) {
|
|
51
|
+
const from = Math.max(startSeconds, reachedTo);
|
|
52
|
+
if (endSeconds > from)
|
|
53
|
+
covered += endSeconds - from;
|
|
54
|
+
reachedTo = Math.max(reachedTo, endSeconds);
|
|
55
|
+
}
|
|
56
|
+
return covered;
|
|
57
|
+
}
|
|
58
|
+
export function hashPrefix(videoId) {
|
|
59
|
+
return createHash('sha256').update(videoId).digest('hex').slice(0, HASH_PREFIX_LENGTH);
|
|
60
|
+
}
|
|
61
|
+
function failure(message, cause) {
|
|
62
|
+
return new YouTubeError('FETCH_FAILED', message, {
|
|
63
|
+
nextStep: 'SponsorBlock is a community service and may be down. Try again later.',
|
|
64
|
+
retryable: true,
|
|
65
|
+
cause,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* The body as text, read as it arrives and abandoned once it passes the cap, so a
|
|
70
|
+
* misbehaving service cannot make this process hold an unbounded response in memory.
|
|
71
|
+
*/
|
|
72
|
+
async function readBounded(response) {
|
|
73
|
+
if (response.body === null)
|
|
74
|
+
return '';
|
|
75
|
+
const reader = response.body.getReader();
|
|
76
|
+
const decoder = new TextDecoder();
|
|
77
|
+
let text = '';
|
|
78
|
+
for (;;) {
|
|
79
|
+
const { done, value } = await reader.read();
|
|
80
|
+
if (done)
|
|
81
|
+
return text + decoder.decode();
|
|
82
|
+
text += decoder.decode(value, { stream: true });
|
|
83
|
+
if (text.length > MAX_RESPONSE_CHARS) {
|
|
84
|
+
await reader.cancel();
|
|
85
|
+
throw failure('SponsorBlock answered with too much data.');
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
async function request(url, timeoutMs) {
|
|
90
|
+
const timeout = AbortSignal.timeout(timeoutMs);
|
|
91
|
+
const { signal: cancelled } = currentContext();
|
|
92
|
+
const signal = cancelled === undefined ? timeout : AbortSignal.any([cancelled, timeout]);
|
|
93
|
+
try {
|
|
94
|
+
// The host is fixed, so a redirect is never legitimate: refusing it keeps this
|
|
95
|
+
// request on the one origin it was written for.
|
|
96
|
+
const response = await fetch(url, {
|
|
97
|
+
signal,
|
|
98
|
+
redirect: 'error',
|
|
99
|
+
headers: { Accept: 'application/json' },
|
|
100
|
+
});
|
|
101
|
+
return { status: response.status, text: await readBounded(response) };
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
if (error instanceof YouTubeError)
|
|
105
|
+
throw error;
|
|
106
|
+
if (error instanceof DOMException && error.name === 'TimeoutError') {
|
|
107
|
+
throw new YouTubeError('TIMEOUT', 'SponsorBlock did not answer in time.', {
|
|
108
|
+
nextStep: 'Try again later.',
|
|
109
|
+
retryable: true,
|
|
110
|
+
cause: error,
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
if (cancelled?.aborted === true) {
|
|
114
|
+
throw new YouTubeError('CANCELLED', 'The request was cancelled.');
|
|
115
|
+
}
|
|
116
|
+
throw failure('Could not reach SponsorBlock.', error);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
export async function fetchSponsorSegments(videoId, categories = SPONSOR_CATEGORIES, options = {}) {
|
|
120
|
+
assertVideoId(videoId);
|
|
121
|
+
const url = new URL(`${API}/${hashPrefix(videoId)}`);
|
|
122
|
+
url.searchParams.set('categories', JSON.stringify(categories));
|
|
123
|
+
const { status, text } = await request(url, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
124
|
+
// 404 is how the service says no video sharing the prefix has any segment.
|
|
125
|
+
if (status === 404)
|
|
126
|
+
return [];
|
|
127
|
+
if (status < 200 || status >= 300) {
|
|
128
|
+
throw failure(`SponsorBlock answered with status ${String(status)}.`);
|
|
129
|
+
}
|
|
130
|
+
let json;
|
|
131
|
+
try {
|
|
132
|
+
json = JSON.parse(text);
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
throw new YouTubeError('MALFORMED_RESPONSE', 'SponsorBlock answered with something that is not JSON.', { cause: error });
|
|
136
|
+
}
|
|
137
|
+
const parsed = responseSchema.safeParse(json);
|
|
138
|
+
if (!parsed.success) {
|
|
139
|
+
throw new YouTubeError('MALFORMED_RESPONSE', 'SponsorBlock answered in a shape this server does not know.', { cause: parsed.error });
|
|
140
|
+
}
|
|
141
|
+
return parsed.data
|
|
142
|
+
.filter((entry) => entry.videoID === videoId)
|
|
143
|
+
.flatMap((entry) => entry.segments)
|
|
144
|
+
.map(({ category, segment: [start, end], votes }) => ({
|
|
145
|
+
category,
|
|
146
|
+
startSeconds: start,
|
|
147
|
+
endSeconds: end,
|
|
148
|
+
durationSeconds: end - start,
|
|
149
|
+
votes: votes ?? 0,
|
|
150
|
+
}))
|
|
151
|
+
.sort((a, b) => a.startSeconds - b.startSeconds);
|
|
152
|
+
}
|
|
153
|
+
//# sourceMappingURL=sponsorblock.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sponsorblock.js","sourceRoot":"","sources":["../../src/utils/sponsorblock.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAE9C;;;;;;;;;GASG;AAEH,MAAM,GAAG,GAAG,2CAA2C,CAAC;AACxD,MAAM,kBAAkB,GAAG,CAAC,CAAC;AAC7B,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,8FAA8F;AAC9F,MAAM,kBAAkB,GAAG,SAAS,CAAC;AAErC,0FAA0F;AAC1F,MAAM,CAAC,MAAM,kBAAkB,GAAG;IAChC,SAAS;IACT,WAAW;IACX,aAAa;IACb,OAAO;IACP,OAAO;IACP,SAAS;IACT,gBAAgB;IAChB,QAAQ;CACA,CAAC;AAaX,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAC5B,CAAC,CAAC,MAAM,CAAC;IACP,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,QAAQ,EAAE,CAAC,CAAC,KAAK,CACf,CAAC,CAAC,MAAM,CAAC;QACP,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;QACpB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1C,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;KAC7B,CAAC,CACH;CACF,CAAC,CACH,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAC,QAAmC;IAChE,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,SAAS,GAAG,MAAM,CAAC,iBAAiB,CAAC;IAEzC,KAAK,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAC3D,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAC1C,EAAE,CAAC;QACF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAC/C,IAAI,UAAU,GAAG,IAAI;YAAE,OAAO,IAAI,UAAU,GAAG,IAAI,CAAC;QACpD,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,OAAe;IACxC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC;AACzF,CAAC;AAED,SAAS,OAAO,CAAC,OAAe,EAAE,KAAe;IAC/C,OAAO,IAAI,YAAY,CAAC,cAAc,EAAE,OAAO,EAAE;QAC/C,QAAQ,EAAE,uEAAuE;QACjF,SAAS,EAAE,IAAI;QACf,KAAK;KACN,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,WAAW,CAAC,QAAkB;IAC3C,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI;QAAE,OAAO,EAAE,CAAC;IAEtC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,IAAI,IAAI,GAAG,EAAE,CAAC;IAEd,SAAS,CAAC;QACR,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,IAAI;YAAE,OAAO,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QACzC,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;YACrC,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,MAAM,OAAO,CAAC,2CAA2C,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,GAAQ,EAAE,SAAiB;IAChD,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/C,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,cAAc,EAAE,CAAC;IAC/C,MAAM,MAAM,GAAG,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;IAEzF,IAAI,CAAC;QACH,+EAA+E;QAC/E,gDAAgD;QAChD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAChC,MAAM;YACN,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE;SACxC,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC;IACxE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY;YAAE,MAAM,KAAK,CAAC;QAC/C,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YACnE,MAAM,IAAI,YAAY,CAAC,SAAS,EAAE,sCAAsC,EAAE;gBACxE,QAAQ,EAAE,kBAAkB;gBAC5B,SAAS,EAAE,IAAI;gBACf,KAAK,EAAE,KAAK;aACb,CAAC,CAAC;QACL,CAAC;QACD,IAAI,SAAS,EAAE,OAAO,KAAK,IAAI,EAAE,CAAC;YAChC,MAAM,IAAI,YAAY,CAAC,WAAW,EAAE,4BAA4B,CAAC,CAAC;QACpE,CAAC;QACD,MAAM,OAAO,CAAC,+BAA+B,EAAE,KAAK,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAe,EACf,aAAgC,kBAAkB,EAClD,UAAkC,EAAE;IAEpC,aAAa,CAAC,OAAO,CAAC,CAAC;IAEvB,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACrD,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC;IAE/D,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;IAErF,2EAA2E;IAC3E,IAAI,MAAM,KAAK,GAAG;QAAE,OAAO,EAAE,CAAC;IAC9B,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QAClC,MAAM,OAAO,CAAC,qCAAqC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,YAAY,CACpB,oBAAoB,EACpB,wDAAwD,EACxD,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,YAAY,CACpB,oBAAoB,EACpB,6DAA6D,EAC7D,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CACxB,CAAC;IACJ,CAAC;IAED,OAAO,MAAM,CAAC,IAAI;SACf,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,OAAO,CAAC;SAC5C,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC;SAClC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QACpD,QAAQ;QACR,YAAY,EAAE,KAAK;QACnB,UAAU,EAAE,GAAG;QACf,eAAe,EAAE,GAAG,GAAG,KAAK;QAC5B,KAAK,EAAE,KAAK,IAAI,CAAC;KAClB,CAAC,CAAC;SACF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;AACrD,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What can be said about the store without harvesting anything.
|
|
3
|
+
*
|
|
4
|
+
* Integrity is checked here and nowhere else on the hot path: `quick_check` is
|
|
5
|
+
* bounded and catches the realistic failures, while the full `integrity_check`
|
|
6
|
+
* is O(database) and would add seconds to every server start on a store that
|
|
7
|
+
* is measured in gigabytes.
|
|
8
|
+
*/
|
|
9
|
+
export interface StoreHealth {
|
|
10
|
+
enabled: boolean;
|
|
11
|
+
path: string;
|
|
12
|
+
exists: boolean;
|
|
13
|
+
storeVersion?: number;
|
|
14
|
+
sizeBytes: number;
|
|
15
|
+
walBytes: number;
|
|
16
|
+
integrity: 'ok' | 'failed' | 'unchecked';
|
|
17
|
+
channels: number;
|
|
18
|
+
videos: number;
|
|
19
|
+
comments: number;
|
|
20
|
+
receipts: {
|
|
21
|
+
complete: number;
|
|
22
|
+
partial: number;
|
|
23
|
+
running: number;
|
|
24
|
+
failed: number;
|
|
25
|
+
};
|
|
26
|
+
error?: string;
|
|
27
|
+
}
|
|
28
|
+
export declare function readStoreHealth(): Promise<StoreHealth>;
|
|
29
|
+
/**
|
|
30
|
+
* Whether the store should make an overall health report say "not ready".
|
|
31
|
+
*
|
|
32
|
+
* A missing store must not: a fresh install has none, and check_health's job
|
|
33
|
+
* is diagnosing the external binaries. A store that fails its integrity check,
|
|
34
|
+
* or claims a version this build does not understand, must.
|
|
35
|
+
*/
|
|
36
|
+
export declare function storeFailsHealth(health: StoreHealth): boolean;
|
|
37
|
+
//# sourceMappingURL=store-health.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store-health.d.ts","sourceRoot":"","sources":["../../src/utils/store-health.ts"],"names":[],"mappings":"AAOA;;;;;;;GAOG;AAEH,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,IAAI,GAAG,QAAQ,GAAG,WAAW,CAAC;IACzC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IACjF,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAiBD,wBAAsB,eAAe,IAAI,OAAO,CAAC,WAAW,CAAC,CA0D5D;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAG7D"}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { stat } from 'node:fs/promises';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { countRows, queryRow } from './store-rows.js';
|
|
4
|
+
import { getStore, readUserVersion } from './store.js';
|
|
5
|
+
import { storeDatabasePath } from './store-paths.js';
|
|
6
|
+
import { STORE_VERSION } from './store-schema.js';
|
|
7
|
+
const quickCheckSchema = z.object({ quick_check: z.string() }).passthrough();
|
|
8
|
+
const stateCountSchema = z.object({ state: z.string(), n: z.number() }).passthrough();
|
|
9
|
+
async function sizeOf(path) {
|
|
10
|
+
try {
|
|
11
|
+
return (await stat(path)).size;
|
|
12
|
+
}
|
|
13
|
+
catch {
|
|
14
|
+
return 0;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
function emptyReceipts() {
|
|
18
|
+
return { complete: 0, partial: 0, running: 0, failed: 0 };
|
|
19
|
+
}
|
|
20
|
+
export async function readStoreHealth() {
|
|
21
|
+
const path = storeDatabasePath();
|
|
22
|
+
const [sizeBytes, walBytes] = await Promise.all([sizeOf(path), sizeOf(`${path}-wal`)]);
|
|
23
|
+
const base = {
|
|
24
|
+
enabled: true,
|
|
25
|
+
path,
|
|
26
|
+
exists: sizeBytes > 0,
|
|
27
|
+
sizeBytes,
|
|
28
|
+
// A -wal file that never shrinks means a reader is pinning it, which is
|
|
29
|
+
// worth seeing before it becomes a disk-space surprise.
|
|
30
|
+
walBytes,
|
|
31
|
+
integrity: 'unchecked',
|
|
32
|
+
channels: 0,
|
|
33
|
+
videos: 0,
|
|
34
|
+
comments: 0,
|
|
35
|
+
receipts: emptyReceipts(),
|
|
36
|
+
};
|
|
37
|
+
try {
|
|
38
|
+
const store = await getStore();
|
|
39
|
+
const quick = queryRow(store.prepare('PRAGMA quick_check'), quickCheckSchema);
|
|
40
|
+
const receipts = emptyReceipts();
|
|
41
|
+
for (const row of store
|
|
42
|
+
.prepare('SELECT state, COUNT(*) AS n FROM harvest_receipt GROUP BY state')
|
|
43
|
+
.all()) {
|
|
44
|
+
const parsed = stateCountSchema.safeParse(row);
|
|
45
|
+
if (!parsed.success)
|
|
46
|
+
continue;
|
|
47
|
+
// A `state` this build does not know about is counted nowhere rather
|
|
48
|
+
// than crashing: the enum is allowed to grow ahead of this reader.
|
|
49
|
+
const { state, n } = parsed.data;
|
|
50
|
+
if (state === 'complete')
|
|
51
|
+
receipts.complete = n;
|
|
52
|
+
else if (state === 'partial')
|
|
53
|
+
receipts.partial = n;
|
|
54
|
+
else if (state === 'running')
|
|
55
|
+
receipts.running = n;
|
|
56
|
+
else if (state === 'failed')
|
|
57
|
+
receipts.failed = n;
|
|
58
|
+
}
|
|
59
|
+
return {
|
|
60
|
+
...base,
|
|
61
|
+
storeVersion: readUserVersion(store),
|
|
62
|
+
integrity: quick?.quick_check === 'ok' ? 'ok' : 'failed',
|
|
63
|
+
channels: countRows(store.prepare('SELECT COUNT(*) FROM channel')),
|
|
64
|
+
videos: countRows(store.prepare('SELECT COUNT(*) FROM video')),
|
|
65
|
+
comments: countRows(store.prepare('SELECT COUNT(*) FROM comment')),
|
|
66
|
+
receipts,
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
// A store that cannot be opened is a reportable condition, not a reason
|
|
71
|
+
// for check_health itself to fail: the binaries it exists to diagnose are
|
|
72
|
+
// unaffected.
|
|
73
|
+
return {
|
|
74
|
+
...base,
|
|
75
|
+
enabled: false,
|
|
76
|
+
error: error instanceof Error ? error.message : String(error),
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* Whether the store should make an overall health report say "not ready".
|
|
82
|
+
*
|
|
83
|
+
* A missing store must not: a fresh install has none, and check_health's job
|
|
84
|
+
* is diagnosing the external binaries. A store that fails its integrity check,
|
|
85
|
+
* or claims a version this build does not understand, must.
|
|
86
|
+
*/
|
|
87
|
+
export function storeFailsHealth(health) {
|
|
88
|
+
if (health.integrity === 'failed')
|
|
89
|
+
return true;
|
|
90
|
+
return health.storeVersion !== undefined && health.storeVersion > STORE_VERSION;
|
|
91
|
+
}
|
|
92
|
+
//# sourceMappingURL=store-health.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store-health.js","sourceRoot":"","sources":["../../src/utils/store-health.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AA0BlD,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AAC7E,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AAEtF,KAAK,UAAU,MAAM,CAAC,IAAY;IAChC,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED,SAAS,aAAa;IACpB,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;AAC5D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe;IACnC,MAAM,IAAI,GAAG,iBAAiB,EAAE,CAAC;IACjC,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;IAEvF,MAAM,IAAI,GAAgB;QACxB,OAAO,EAAE,IAAI;QACb,IAAI;QACJ,MAAM,EAAE,SAAS,GAAG,CAAC;QACrB,SAAS;QACT,wEAAwE;QACxE,wDAAwD;QACxD,QAAQ;QACR,SAAS,EAAE,WAAW;QACtB,QAAQ,EAAE,CAAC;QACX,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,aAAa,EAAE;KAC1B,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,QAAQ,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,oBAAoB,CAAC,EAAE,gBAAgB,CAAC,CAAC;QAE9E,MAAM,QAAQ,GAAG,aAAa,EAAE,CAAC;QACjC,KAAK,MAAM,GAAG,IAAI,KAAK;aACpB,OAAO,CAAC,iEAAiE,CAAC;aAC1E,GAAG,EAAE,EAAE,CAAC;YACT,MAAM,MAAM,GAAG,gBAAgB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAC/C,IAAI,CAAC,MAAM,CAAC,OAAO;gBAAE,SAAS;YAE9B,qEAAqE;YACrE,mEAAmE;YACnE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC;YACjC,IAAI,KAAK,KAAK,UAAU;gBAAE,QAAQ,CAAC,QAAQ,GAAG,CAAC,CAAC;iBAC3C,IAAI,KAAK,KAAK,SAAS;gBAAE,QAAQ,CAAC,OAAO,GAAG,CAAC,CAAC;iBAC9C,IAAI,KAAK,KAAK,SAAS;gBAAE,QAAQ,CAAC,OAAO,GAAG,CAAC,CAAC;iBAC9C,IAAI,KAAK,KAAK,QAAQ;gBAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;QACnD,CAAC;QAED,OAAO;YACL,GAAG,IAAI;YACP,YAAY,EAAE,eAAe,CAAC,KAAK,CAAC;YACpC,SAAS,EAAE,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ;YACxD,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC;YAClE,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,4BAA4B,CAAC,CAAC;YAC9D,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC;YAClE,QAAQ;SACT,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,wEAAwE;QACxE,0EAA0E;QAC1E,cAAc;QACd,OAAO;YACL,GAAG,IAAI;YACP,OAAO,EAAE,KAAK;YACd,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SAC9D,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAmB;IAClD,IAAI,MAAM,CAAC,SAAS,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC/C,OAAO,MAAM,CAAC,YAAY,KAAK,SAAS,IAAI,MAAM,CAAC,YAAY,GAAG,aAAa,CAAC;AAClF,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { DatabaseSync } from 'node:sqlite';
|
|
2
|
+
/**
|
|
3
|
+
* Ordered schema migrations for the harvested store.
|
|
4
|
+
*
|
|
5
|
+
* `PRAGMA user_version` carries the store's version, deliberately independent
|
|
6
|
+
* of the server's semver: the schema changes far less often than the code, and
|
|
7
|
+
* tying them together would force an empty migration on every release.
|
|
8
|
+
*
|
|
9
|
+
* Version 0 means "no store yet". Creating the tables is `STORE_SCHEMA`'s job,
|
|
10
|
+
* not a migration's, so the first run only stamps the version.
|
|
11
|
+
*/
|
|
12
|
+
export interface StoreMigration {
|
|
13
|
+
/** The version this migration produces. */
|
|
14
|
+
to: number;
|
|
15
|
+
apply: (database: DatabaseSync) => void;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Empty on purpose at v1: `STORE_SCHEMA` is `CREATE TABLE IF NOT EXISTS`
|
|
19
|
+
* throughout, so a fresh database and an existing v1 database converge without
|
|
20
|
+
* one. The list exists so the second schema change is a one-line addition
|
|
21
|
+
* rather than a retrofit.
|
|
22
|
+
*/
|
|
23
|
+
export declare const STORE_MIGRATIONS: readonly StoreMigration[];
|
|
24
|
+
/**
|
|
25
|
+
* Brings a database up to STORE_VERSION.
|
|
26
|
+
*
|
|
27
|
+
* Each migration runs inside its own transaction so a failure leaves the
|
|
28
|
+
* store at the last version that fully applied, rather than half-way through
|
|
29
|
+
* one. A newer-than-known version is rejected before this is reached, in
|
|
30
|
+
* `store.ts`.
|
|
31
|
+
*/
|
|
32
|
+
export declare function runStoreMigrations(database: DatabaseSync, fromVersion: number): number;
|
|
33
|
+
//# sourceMappingURL=store-migrations.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store-migrations.d.ts","sourceRoot":"","sources":["../../src/utils/store-migrations.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAIhD;;;;;;;;;GASG;AAEH,MAAM,WAAW,cAAc;IAC7B,2CAA2C;IAC3C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CACzC;AAED;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,EAAE,SAAS,cAAc,EAAO,CAAC;AAE9D;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAwBtF"}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { STORE_VERSION } from './store-schema.js';
|
|
2
|
+
import { writeUserVersion } from './store.js';
|
|
3
|
+
/**
|
|
4
|
+
* Empty on purpose at v1: `STORE_SCHEMA` is `CREATE TABLE IF NOT EXISTS`
|
|
5
|
+
* throughout, so a fresh database and an existing v1 database converge without
|
|
6
|
+
* one. The list exists so the second schema change is a one-line addition
|
|
7
|
+
* rather than a retrofit.
|
|
8
|
+
*/
|
|
9
|
+
export const STORE_MIGRATIONS = [];
|
|
10
|
+
/**
|
|
11
|
+
* Brings a database up to STORE_VERSION.
|
|
12
|
+
*
|
|
13
|
+
* Each migration runs inside its own transaction so a failure leaves the
|
|
14
|
+
* store at the last version that fully applied, rather than half-way through
|
|
15
|
+
* one. A newer-than-known version is rejected before this is reached, in
|
|
16
|
+
* `store.ts`.
|
|
17
|
+
*/
|
|
18
|
+
export function runStoreMigrations(database, fromVersion) {
|
|
19
|
+
let current = fromVersion;
|
|
20
|
+
for (const migration of STORE_MIGRATIONS) {
|
|
21
|
+
if (migration.to <= current)
|
|
22
|
+
continue;
|
|
23
|
+
database.exec('BEGIN');
|
|
24
|
+
try {
|
|
25
|
+
migration.apply(database);
|
|
26
|
+
writeUserVersion(database, migration.to);
|
|
27
|
+
database.exec('COMMIT');
|
|
28
|
+
}
|
|
29
|
+
catch (error) {
|
|
30
|
+
database.exec('ROLLBACK');
|
|
31
|
+
throw error;
|
|
32
|
+
}
|
|
33
|
+
current = migration.to;
|
|
34
|
+
}
|
|
35
|
+
if (current !== STORE_VERSION) {
|
|
36
|
+
writeUserVersion(database, STORE_VERSION);
|
|
37
|
+
current = STORE_VERSION;
|
|
38
|
+
}
|
|
39
|
+
return current;
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=store-migrations.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"store-migrations.js","sourceRoot":"","sources":["../../src/utils/store-migrations.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAmB9C;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAA8B,EAAE,CAAC;AAE9D;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAsB,EAAE,WAAmB;IAC5E,IAAI,OAAO,GAAG,WAAW,CAAC;IAE1B,KAAK,MAAM,SAAS,IAAI,gBAAgB,EAAE,CAAC;QACzC,IAAI,SAAS,CAAC,EAAE,IAAI,OAAO;YAAE,SAAS;QAEtC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,IAAI,CAAC;YACH,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC1B,gBAAgB,CAAC,QAAQ,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC1B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1B,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,GAAG,SAAS,CAAC,EAAE,CAAC;IACzB,CAAC;IAED,IAAI,OAAO,KAAK,aAAa,EAAE,CAAC;QAC9B,gBAAgB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QAC1C,OAAO,GAAG,aAAa,CAAC;IAC1B,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|