prismcast 1.12.0 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app.d.ts +41 -1
- package/dist/app.js +164 -76
- package/dist/app.js.map +1 -1
- package/dist/browser/blockedPage.d.ts +0 -1
- package/dist/browser/browserSupervisor.d.ts +34 -5
- package/dist/browser/browserSupervisor.js +39 -12
- package/dist/browser/browserSupervisor.js.map +1 -1
- package/dist/browser/cdp.d.ts +58 -6
- package/dist/browser/cdp.js +147 -9
- package/dist/browser/cdp.js.map +1 -1
- package/dist/browser/channelSelection.d.ts +18 -6
- package/dist/browser/channelSelection.js +47 -36
- package/dist/browser/channelSelection.js.map +1 -1
- package/dist/browser/consent.d.ts +33 -4
- package/dist/browser/consent.js +87 -50
- package/dist/browser/consent.js.map +1 -1
- package/dist/browser/display.d.ts +0 -1
- package/dist/browser/hlsPlaylistObserver.d.ts +3 -2
- package/dist/browser/hlsPlaylistObserver.js +11 -6
- package/dist/browser/hlsPlaylistObserver.js.map +1 -1
- package/dist/browser/index.d.ts +273 -59
- package/dist/browser/index.js +768 -255
- package/dist/browser/index.js.map +1 -1
- package/dist/browser/launchGovernor.d.ts +0 -1
- package/dist/browser/login.d.ts +11 -10
- package/dist/browser/login.js +42 -33
- package/dist/browser/login.js.map +1 -1
- package/dist/browser/manifestInterceptor.d.ts +14 -5
- package/dist/browser/manifestInterceptor.js +26 -13
- package/dist/browser/manifestInterceptor.js.map +1 -1
- package/dist/browser/navigation.d.ts +28 -0
- package/dist/browser/navigation.js +57 -0
- package/dist/browser/navigation.js.map +1 -0
- package/dist/browser/pageStaleness.d.ts +1 -2
- package/dist/browser/pageStaleness.js +9 -9
- package/dist/browser/pageStaleness.js.map +1 -1
- package/dist/browser/precaching.d.ts +27 -13
- package/dist/browser/precaching.js +219 -73
- package/dist/browser/precaching.js.map +1 -1
- package/dist/browser/tabCapture.d.ts +97 -0
- package/dist/browser/tabCapture.js +314 -0
- package/dist/browser/tabCapture.js.map +1 -0
- package/dist/browser/tabNetworkObserver.d.ts +0 -1
- package/dist/browser/tabSelection.d.ts +117 -0
- package/dist/browser/tabSelection.js +547 -0
- package/dist/browser/tabSelection.js.map +1 -0
- package/dist/browser/tuning/cache.d.ts +0 -1
- package/dist/browser/tuning/comcastPolymer.d.ts +0 -1
- package/dist/browser/tuning/comcastPolymer.js +18 -19
- package/dist/browser/tuning/comcastPolymer.js.map +1 -1
- package/dist/browser/tuning/cox.d.ts +0 -1
- package/dist/browser/tuning/directv.d.ts +0 -1
- package/dist/browser/tuning/directv.js +12 -13
- package/dist/browser/tuning/directv.js.map +1 -1
- package/dist/browser/tuning/fox.d.ts +0 -1
- package/dist/browser/tuning/fox.js +2 -2
- package/dist/browser/tuning/fox.js.map +1 -1
- package/dist/browser/tuning/gridSearch.d.ts +0 -1
- package/dist/browser/tuning/hbo.d.ts +0 -1
- package/dist/browser/tuning/hbo.js +2 -2
- package/dist/browser/tuning/hbo.js.map +1 -1
- package/dist/browser/tuning/hulu.d.ts +14 -2
- package/dist/browser/tuning/hulu.js +150 -24
- package/dist/browser/tuning/hulu.js.map +1 -1
- package/dist/browser/tuning/shared.d.ts +1 -2
- package/dist/browser/tuning/shared.js +1 -1
- package/dist/browser/tuning/sling.d.ts +0 -1
- package/dist/browser/tuning/sling.js +17 -22
- package/dist/browser/tuning/sling.js.map +1 -1
- package/dist/browser/tuning/spectrum.d.ts +0 -1
- package/dist/browser/tuning/thumbnailRow.d.ts +0 -1
- package/dist/browser/tuning/tileClick.d.ts +0 -1
- package/dist/browser/tuning/xfinity.d.ts +0 -1
- package/dist/browser/tuning/youtubeTv.d.ts +0 -1
- package/dist/browser/tuning/youtubeTv.js +1 -2
- package/dist/browser/tuning/youtubeTv.js.map +1 -1
- package/dist/browser/video.d.ts +55 -33
- package/dist/browser/video.js +213 -169
- package/dist/browser/video.js.map +1 -1
- package/dist/browser/windowSync.d.ts +45 -0
- package/dist/browser/windowSync.js +153 -0
- package/dist/browser/windowSync.js.map +1 -0
- package/dist/channels/index.d.ts +7 -1
- package/dist/channels/index.js +24 -8
- package/dist/channels/index.js.map +1 -1
- package/dist/config/channelForm.d.ts +0 -1
- package/dist/config/channelIdentity.d.ts +27 -0
- package/dist/config/channelIdentity.js +72 -0
- package/dist/config/channelIdentity.js.map +1 -0
- package/dist/config/channelSort.d.ts +25 -0
- package/dist/config/channelSort.js +101 -0
- package/dist/config/channelSort.js.map +1 -0
- package/dist/config/consistencyProbe.d.ts +0 -1
- package/dist/config/consistencyProbe.js +1 -1
- package/dist/config/consistencyProbe.js.map +1 -1
- package/dist/config/health.d.ts +3 -2
- package/dist/config/health.js +41 -31
- package/dist/config/health.js.map +1 -1
- package/dist/config/index.d.ts +12 -1
- package/dist/config/index.js +77 -31
- package/dist/config/index.js.map +1 -1
- package/dist/config/paths.d.ts +17 -2
- package/dist/config/paths.js +21 -1
- package/dist/config/paths.js.map +1 -1
- package/dist/config/persistence.context.d.ts +0 -1
- package/dist/config/persistence.d.ts +8 -4
- package/dist/config/persistence.js +143 -82
- package/dist/config/persistence.js.map +1 -1
- package/dist/config/presets.d.ts +0 -1
- package/dist/config/profiles.d.ts +9 -4
- package/dist/config/profiles.js +39 -6
- package/dist/config/profiles.js.map +1 -1
- package/dist/config/providerLineups.d.ts +0 -1
- package/dist/config/providerLineups.js +0 -3
- package/dist/config/providerLineups.js.map +1 -1
- package/dist/config/reactivity.d.ts +0 -1
- package/dist/config/servicePacks.d.ts +0 -1
- package/dist/config/services.d.ts +1 -47
- package/dist/config/services.js +4 -135
- package/dist/config/services.js.map +1 -1
- package/dist/config/sites.d.ts +0 -9
- package/dist/config/sites.js +82 -98
- package/dist/config/sites.js.map +1 -1
- package/dist/config/userChannels.d.ts +22 -29
- package/dist/config/userChannels.js +47 -65
- package/dist/config/userChannels.js.map +1 -1
- package/dist/config/userConfig.d.ts +19 -4
- package/dist/config/userConfig.js +123 -25
- package/dist/config/userConfig.js.map +1 -1
- package/dist/config/userProfiles.d.ts +2 -3
- package/dist/config/userProfiles.js +16 -13
- package/dist/config/userProfiles.js.map +1 -1
- package/dist/hdhr/channelMap.d.ts +0 -1
- package/dist/hdhr/deviceId.d.ts +0 -1
- package/dist/hdhr/discover.d.ts +0 -1
- package/dist/hdhr/getHandlers.d.ts +0 -1
- package/dist/hdhr/identity.d.ts +0 -1
- package/dist/hdhr/index.d.ts +0 -1
- package/dist/hdhr/protocol.d.ts +1 -2
- package/dist/hdhr/protocol.js +1 -1
- package/dist/hdhr/tunerState.d.ts +0 -1
- package/dist/hdhr/udp.d.ts +0 -1
- package/dist/hdhr/udp.js +16 -4
- package/dist/hdhr/udp.js.map +1 -1
- package/dist/identity.d.ts +0 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.js +10 -3
- package/dist/index.js.map +1 -1
- package/dist/native/codecInference.d.ts +5 -4
- package/dist/native/codecInference.js +12 -7
- package/dist/native/codecInference.js.map +1 -1
- package/dist/native/decrypt.d.ts +10 -3
- package/dist/native/decrypt.js +9 -5
- package/dist/native/decrypt.js.map +1 -1
- package/dist/native/index.d.ts +26 -24
- package/dist/native/index.js +26 -20
- package/dist/native/index.js.map +1 -1
- package/dist/native/probe.d.ts +7 -10
- package/dist/native/probe.js +41 -37
- package/dist/native/probe.js.map +1 -1
- package/dist/native/proxy.d.ts +3 -4
- package/dist/native/proxy.js +91 -69
- package/dist/native/proxy.js.map +1 -1
- package/dist/native/tokenExpiry.d.ts +0 -1
- package/dist/routes/assets.d.ts +0 -1
- package/dist/routes/auth.d.ts +0 -1
- package/dist/routes/auth.js +1 -1
- package/dist/routes/auth.js.map +1 -1
- package/dist/routes/cdp.d.ts +11 -5
- package/dist/routes/cdp.js +91 -63
- package/dist/routes/cdp.js.map +1 -1
- package/dist/routes/channels.d.ts +0 -1
- package/dist/routes/clientActions.d.ts +0 -1
- package/dist/routes/components.d.ts +0 -1
- package/dist/routes/config/channels/endpoints/browse.d.ts +0 -1
- package/dist/routes/config/channels/endpoints/bulk.d.ts +0 -1
- package/dist/routes/config/channels/endpoints/bulk.js +3 -3
- package/dist/routes/config/channels/endpoints/bulk.js.map +1 -1
- package/dist/routes/config/channels/endpoints/crud.d.ts +0 -1
- package/dist/routes/config/channels/endpoints/crud.js +4 -3
- package/dist/routes/config/channels/endpoints/crud.js.map +1 -1
- package/dist/routes/config/channels/endpoints/importExport.d.ts +0 -1
- package/dist/routes/config/channels/endpoints/importExport.js +7 -3
- package/dist/routes/config/channels/endpoints/importExport.js.map +1 -1
- package/dist/routes/config/channels/endpoints/predefined.d.ts +0 -1
- package/dist/routes/config/channels/endpoints/prefs.d.ts +0 -1
- package/dist/routes/config/channels/endpoints/prefs.js +1 -1
- package/dist/routes/config/channels/endpoints/prefs.js.map +1 -1
- package/dist/routes/config/channels/endpoints/service.d.ts +0 -1
- package/dist/routes/config/channels/endpoints/service.js +8 -5
- package/dist/routes/config/channels/endpoints/service.js.map +1 -1
- package/dist/routes/config/channels/endpoints/tags.d.ts +0 -1
- package/dist/routes/config/channels/endpoints/tags.js +3 -3
- package/dist/routes/config/channels/endpoints/tags.js.map +1 -1
- package/dist/routes/config/channels/healthBridge.d.ts +4 -1
- package/dist/routes/config/channels/healthBridge.js +9 -5
- package/dist/routes/config/channels/healthBridge.js.map +1 -1
- package/dist/routes/config/channels/http/handler.d.ts +0 -1
- package/dist/routes/config/channels/http/playlistHint.d.ts +0 -1
- package/dist/routes/config/channels/http/serviceWarning.d.ts +0 -1
- package/dist/routes/config/channels/http/serviceWarning.js +1 -1
- package/dist/routes/config/channels/http/serviceWarning.js.map +1 -1
- package/dist/routes/config/channels/index.d.ts +0 -1
- package/dist/routes/config/channels/setup.d.ts +0 -1
- package/dist/routes/config/channels/table.d.ts +0 -1
- package/dist/routes/config/channels/table.js +66 -134
- package/dist/routes/config/channels/table.js.map +1 -1
- package/dist/routes/config/http/envelope.d.ts +1 -2
- package/dist/routes/config/index.d.ts +5 -5
- package/dist/routes/config/index.js +7 -5
- package/dist/routes/config/index.js.map +1 -1
- package/dist/routes/config/services.d.ts +0 -1
- package/dist/routes/config/services.js +17 -19
- package/dist/routes/config/services.js.map +1 -1
- package/dist/routes/config/settings.d.ts +6 -15
- package/dist/routes/config/settings.js +16 -35
- package/dist/routes/config/settings.js.map +1 -1
- package/dist/routes/debug.d.ts +0 -1
- package/dist/routes/debug.js +1 -1
- package/dist/routes/debug.js.map +1 -1
- package/dist/routes/health.d.ts +16 -8
- package/dist/routes/health.js +27 -12
- package/dist/routes/health.js.map +1 -1
- package/dist/routes/hls.d.ts +0 -1
- package/dist/routes/icons.d.ts +0 -1
- package/dist/routes/index.d.ts +0 -1
- package/dist/routes/logs.d.ts +3 -2
- package/dist/routes/logs.js +54 -14
- package/dist/routes/logs.js.map +1 -1
- package/dist/routes/mpegts.d.ts +0 -1
- package/dist/routes/play.d.ts +0 -1
- package/dist/routes/playlist.d.ts +0 -1
- package/dist/routes/playlist.js +2 -1
- package/dist/routes/playlist.js.map +1 -1
- package/dist/routes/root/content.d.ts +0 -1
- package/dist/routes/root/content.js +32 -11
- package/dist/routes/root/content.js.map +1 -1
- package/dist/routes/root/index.d.ts +10 -1
- package/dist/routes/root/index.js +17 -7
- package/dist/routes/root/index.js.map +1 -1
- package/dist/routes/root/scripts/channels.d.ts +0 -1
- package/dist/routes/root/scripts/channels.js +10 -20
- package/dist/routes/root/scripts/channels.js.map +1 -1
- package/dist/routes/root/scripts/clientEscape.d.ts +0 -1
- package/dist/routes/root/scripts/clientEscape.js +1 -1
- package/dist/routes/root/scripts/clientUrl.d.ts +0 -1
- package/dist/routes/root/scripts/config.d.ts +0 -1
- package/dist/routes/root/scripts/config.js +8 -29
- package/dist/routes/root/scripts/config.js.map +1 -1
- package/dist/routes/root/scripts/index.d.ts +0 -1
- package/dist/routes/root/scripts/shared.d.ts +0 -1
- package/dist/routes/root/scripts/shared.js +16 -13
- package/dist/routes/root/scripts/shared.js.map +1 -1
- package/dist/routes/root/scripts/status.d.ts +0 -1
- package/dist/routes/root/scripts/status.handlers.d.ts +3 -1
- package/dist/routes/root/scripts/status.handlers.js +52 -26
- package/dist/routes/root/scripts/status.handlers.js.map +1 -1
- package/dist/routes/root/styles.d.ts +0 -1
- package/dist/routes/root/styles.js +10 -18
- package/dist/routes/root/styles.js.map +1 -1
- package/dist/routes/services.d.ts +0 -1
- package/dist/routes/services.js +1 -1
- package/dist/routes/services.js.map +1 -1
- package/dist/routes/sse.d.ts +3 -2
- package/dist/routes/sse.js +7 -3
- package/dist/routes/sse.js.map +1 -1
- package/dist/routes/streams.d.ts +4 -2
- package/dist/routes/streams.js +8 -5
- package/dist/routes/streams.js.map +1 -1
- package/dist/routes/theme.d.ts +0 -1
- package/dist/routes/theme.js +3 -3
- package/dist/routes/theme.js.map +1 -1
- package/dist/routes/ui.d.ts +0 -1
- package/dist/routes/ui.js +5 -5
- package/dist/routes/ui.js.map +1 -1
- package/dist/routes/upgrade.d.ts +0 -1
- package/dist/routes/upgrade.js +2 -1
- package/dist/routes/upgrade.js.map +1 -1
- package/dist/service/commands.context.d.ts +0 -1
- package/dist/service/commands.context.js +7 -1
- package/dist/service/commands.context.js.map +1 -1
- package/dist/service/commands.d.ts +0 -1
- package/dist/service/generators.context.d.ts +0 -1
- package/dist/service/generators.d.ts +0 -1
- package/dist/service/index.d.ts +0 -1
- package/dist/streaming/captureLock.d.ts +1 -2
- package/dist/streaming/captureLock.js +8 -7
- package/dist/streaming/captureLock.js.map +1 -1
- package/dist/streaming/captureSession.d.ts +0 -1
- package/dist/streaming/clients.d.ts +4 -3
- package/dist/streaming/clients.js +5 -4
- package/dist/streaming/clients.js.map +1 -1
- package/dist/streaming/codec.d.ts +0 -1
- package/dist/streaming/fmp4Segmenter.d.ts +11 -6
- package/dist/streaming/fmp4Segmenter.js +39 -17
- package/dist/streaming/fmp4Segmenter.js.map +1 -1
- package/dist/streaming/hls.d.ts +96 -3
- package/dist/streaming/hls.js +282 -171
- package/dist/streaming/hls.js.map +1 -1
- package/dist/streaming/hlsResume.d.ts +8 -5
- package/dist/streaming/hlsResume.js +10 -8
- package/dist/streaming/hlsResume.js.map +1 -1
- package/dist/streaming/hlsSegments.d.ts +11 -8
- package/dist/streaming/hlsSegments.js +38 -18
- package/dist/streaming/hlsSegments.js.map +1 -1
- package/dist/streaming/lifecycle.d.ts +5 -4
- package/dist/streaming/lifecycle.js +43 -13
- package/dist/streaming/lifecycle.js.map +1 -1
- package/dist/streaming/monitor.d.ts +14 -3
- package/dist/streaming/monitor.js +447 -234
- package/dist/streaming/monitor.js.map +1 -1
- package/dist/streaming/mp4Parser.d.ts +0 -1
- package/dist/streaming/mpegts.d.ts +11 -1
- package/dist/streaming/mpegts.js +92 -28
- package/dist/streaming/mpegts.js.map +1 -1
- package/dist/streaming/playlistBuilder.d.ts +0 -1
- package/dist/streaming/preroll.d.ts +34 -15
- package/dist/streaming/preroll.js +32 -30
- package/dist/streaming/preroll.js.map +1 -1
- package/dist/streaming/pretune.d.ts +2 -3
- package/dist/streaming/pretune.js +45 -45
- package/dist/streaming/pretune.js.map +1 -1
- package/dist/streaming/pretuneTimers.d.ts +16 -12
- package/dist/streaming/pretuneTimers.js +27 -37
- package/dist/streaming/pretuneTimers.js.map +1 -1
- package/dist/streaming/recovery.d.ts +97 -9
- package/dist/streaming/recovery.js +96 -18
- package/dist/streaming/recovery.js.map +1 -1
- package/dist/streaming/registry.d.ts +102 -15
- package/dist/streaming/registry.js +90 -5
- package/dist/streaming/registry.js.map +1 -1
- package/dist/streaming/setup.d.ts +90 -32
- package/dist/streaming/setup.js +379 -248
- package/dist/streaming/setup.js.map +1 -1
- package/dist/streaming/showInfo.d.ts +8 -4
- package/dist/streaming/showInfo.js +54 -45
- package/dist/streaming/showInfo.js.map +1 -1
- package/dist/streaming/statusEmitter.d.ts +5 -3
- package/dist/streaming/statusEmitter.js +20 -8
- package/dist/streaming/statusEmitter.js.map +1 -1
- package/dist/types/channels.d.ts +0 -1
- package/dist/types/channels.js +0 -4
- package/dist/types/channels.js.map +1 -1
- package/dist/types/config.d.ts +0 -1
- package/dist/types/index.d.ts +4 -4
- package/dist/types/index.js +2 -1
- package/dist/types/index.js.map +1 -1
- package/dist/types/profiles.d.ts +51 -9
- package/dist/types/profiles.js +35 -1
- package/dist/types/profiles.js.map +1 -1
- package/dist/types/selection.d.ts +10 -1
- package/dist/types/shared.d.ts +0 -1
- package/dist/types/streaming.d.ts +8 -1
- package/dist/types/streaming.js +7 -0
- package/dist/types/streaming.js.map +1 -1
- package/dist/upgrade/commands.context.d.ts +0 -1
- package/dist/upgrade/commands.d.ts +0 -1
- package/dist/upgrade/commands.js +1 -1
- package/dist/upgrade/commands.js.map +1 -1
- package/dist/upgrade/detection.context.d.ts +0 -1
- package/dist/upgrade/detection.d.ts +1 -2
- package/dist/upgrade/index.d.ts +0 -1
- package/dist/upgrade/lifecycle.context.d.ts +2 -1
- package/dist/upgrade/lifecycle.context.js +9 -4
- package/dist/upgrade/lifecycle.context.js.map +1 -1
- package/dist/upgrade/lifecycle.d.ts +0 -1
- package/dist/upgrade/pathHandle.d.ts +0 -1
- package/dist/utils/bootSession.context.d.ts +0 -1
- package/dist/utils/bootSession.context.js +3 -2
- package/dist/utils/bootSession.context.js.map +1 -1
- package/dist/utils/bootSession.d.ts +0 -1
- package/dist/utils/chromeFetch.d.ts +0 -1
- package/dist/utils/cliOutput.d.ts +0 -1
- package/dist/utils/debugFilter.d.ts +0 -1
- package/dist/utils/debugFilter.js +5 -3
- package/dist/utils/debugFilter.js.map +1 -1
- package/dist/utils/delay.d.ts +65 -13
- package/dist/utils/delay.js +96 -23
- package/dist/utils/delay.js.map +1 -1
- package/dist/utils/errors.d.ts +0 -11
- package/dist/utils/errors.js +3 -14
- package/dist/utils/errors.js.map +1 -1
- package/dist/utils/evaluate.d.ts +11 -3
- package/dist/utils/evaluate.js +4 -3
- package/dist/utils/evaluate.js.map +1 -1
- package/dist/utils/ffmpeg.context.d.ts +0 -1
- package/dist/utils/ffmpeg.d.ts +1 -1
- package/dist/utils/ffmpeg.js +2 -1
- package/dist/utils/ffmpeg.js.map +1 -1
- package/dist/utils/fileLogger.d.ts +26 -7
- package/dist/utils/fileLogger.js +241 -102
- package/dist/utils/fileLogger.js.map +1 -1
- package/dist/utils/format.d.ts +36 -4
- package/dist/utils/format.js +66 -12
- package/dist/utils/format.js.map +1 -1
- package/dist/utils/index.d.ts +1 -2
- package/dist/utils/index.js +1 -1
- package/dist/utils/index.js.map +1 -1
- package/dist/utils/logEmitter.d.ts +0 -1
- package/dist/utils/logger.d.ts +0 -1
- package/dist/utils/m3u.d.ts +0 -1
- package/dist/utils/markup.d.ts +0 -1
- package/dist/utils/memo.d.ts +0 -1
- package/dist/utils/morganStream.d.ts +2 -3
- package/dist/utils/morganStream.js +6 -8
- package/dist/utils/morganStream.js.map +1 -1
- package/dist/utils/network.d.ts +0 -1
- package/dist/utils/never.d.ts +8 -0
- package/dist/utils/never.js +20 -0
- package/dist/utils/never.js.map +1 -0
- package/dist/utils/pid.d.ts +0 -1
- package/dist/utils/platform.d.ts +0 -1
- package/dist/utils/processInspector.context.d.ts +32 -2
- package/dist/utils/processInspector.context.js +117 -1
- package/dist/utils/processInspector.context.js.map +1 -1
- package/dist/utils/processInspector.d.ts +0 -32
- package/dist/utils/processInspector.js +10 -117
- package/dist/utils/processInspector.js.map +1 -1
- package/dist/utils/retry.d.ts +5 -4
- package/dist/utils/retry.js +43 -33
- package/dist/utils/retry.js.map +1 -1
- package/dist/utils/runtimeIdentity.context.d.ts +0 -1
- package/dist/utils/runtimeIdentity.context.js +3 -1
- package/dist/utils/runtimeIdentity.context.js.map +1 -1
- package/dist/utils/runtimeIdentity.d.ts +1 -1
- package/dist/utils/runtimeIdentity.js +1 -1
- package/dist/utils/runtimeIdentity.js.map +1 -1
- package/dist/utils/sanitize.d.ts +8 -1
- package/dist/utils/sanitize.js +19 -0
- package/dist/utils/sanitize.js.map +1 -1
- package/dist/utils/streamContext.d.ts +0 -1
- package/dist/utils/timing.d.ts +3 -4
- package/dist/utils/timing.js +4 -4
- package/dist/utils/timing.js.map +1 -1
- package/dist/utils/version.d.ts +14 -8
- package/dist/utils/version.js +35 -24
- package/dist/utils/version.js.map +1 -1
- package/package.json +7 -7
- package/dist/app.d.ts.map +0 -1
- package/dist/browser/blockedPage.d.ts.map +0 -1
- package/dist/browser/browserSupervisor.d.ts.map +0 -1
- package/dist/browser/cdp.d.ts.map +0 -1
- package/dist/browser/channelSelection.d.ts.map +0 -1
- package/dist/browser/consent.d.ts.map +0 -1
- package/dist/browser/display.d.ts.map +0 -1
- package/dist/browser/hlsPlaylistObserver.d.ts.map +0 -1
- package/dist/browser/index.d.ts.map +0 -1
- package/dist/browser/launchGovernor.d.ts.map +0 -1
- package/dist/browser/login.d.ts.map +0 -1
- package/dist/browser/manifestInterceptor.d.ts.map +0 -1
- package/dist/browser/pageStaleness.d.ts.map +0 -1
- package/dist/browser/precaching.d.ts.map +0 -1
- package/dist/browser/tabNetworkObserver.d.ts.map +0 -1
- package/dist/browser/tuning/cache.d.ts.map +0 -1
- package/dist/browser/tuning/comcastPolymer.d.ts.map +0 -1
- package/dist/browser/tuning/cox.d.ts.map +0 -1
- package/dist/browser/tuning/directv.d.ts.map +0 -1
- package/dist/browser/tuning/fox.d.ts.map +0 -1
- package/dist/browser/tuning/gridSearch.d.ts.map +0 -1
- package/dist/browser/tuning/hbo.d.ts.map +0 -1
- package/dist/browser/tuning/hulu.d.ts.map +0 -1
- package/dist/browser/tuning/shared.d.ts.map +0 -1
- package/dist/browser/tuning/sling.d.ts.map +0 -1
- package/dist/browser/tuning/spectrum.d.ts.map +0 -1
- package/dist/browser/tuning/thumbnailRow.d.ts.map +0 -1
- package/dist/browser/tuning/tileClick.d.ts.map +0 -1
- package/dist/browser/tuning/xfinity.d.ts.map +0 -1
- package/dist/browser/tuning/youtubeTv.d.ts.map +0 -1
- package/dist/browser/video.d.ts.map +0 -1
- package/dist/channels/index.d.ts.map +0 -1
- package/dist/config/channelForm.d.ts.map +0 -1
- package/dist/config/consistencyProbe.d.ts.map +0 -1
- package/dist/config/health.d.ts.map +0 -1
- package/dist/config/index.d.ts.map +0 -1
- package/dist/config/paths.d.ts.map +0 -1
- package/dist/config/persistence.context.d.ts.map +0 -1
- package/dist/config/persistence.d.ts.map +0 -1
- package/dist/config/presets.d.ts.map +0 -1
- package/dist/config/profiles.d.ts.map +0 -1
- package/dist/config/providerLineups.d.ts.map +0 -1
- package/dist/config/reactivity.d.ts.map +0 -1
- package/dist/config/servicePacks.d.ts.map +0 -1
- package/dist/config/services.d.ts.map +0 -1
- package/dist/config/sites.d.ts.map +0 -1
- package/dist/config/userChannels.d.ts.map +0 -1
- package/dist/config/userConfig.d.ts.map +0 -1
- package/dist/config/userProfiles.d.ts.map +0 -1
- package/dist/hdhr/channelMap.d.ts.map +0 -1
- package/dist/hdhr/deviceId.d.ts.map +0 -1
- package/dist/hdhr/discover.d.ts.map +0 -1
- package/dist/hdhr/getHandlers.d.ts.map +0 -1
- package/dist/hdhr/identity.d.ts.map +0 -1
- package/dist/hdhr/index.d.ts.map +0 -1
- package/dist/hdhr/protocol.d.ts.map +0 -1
- package/dist/hdhr/tunerState.d.ts.map +0 -1
- package/dist/hdhr/udp.d.ts.map +0 -1
- package/dist/identity.d.ts.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/native/codecInference.d.ts.map +0 -1
- package/dist/native/decrypt.d.ts.map +0 -1
- package/dist/native/index.d.ts.map +0 -1
- package/dist/native/probe.d.ts.map +0 -1
- package/dist/native/proxy.d.ts.map +0 -1
- package/dist/native/tokenExpiry.d.ts.map +0 -1
- package/dist/routes/assets.d.ts.map +0 -1
- package/dist/routes/auth.d.ts.map +0 -1
- package/dist/routes/cdp.d.ts.map +0 -1
- package/dist/routes/channels.d.ts.map +0 -1
- package/dist/routes/clientActions.d.ts.map +0 -1
- package/dist/routes/components.d.ts.map +0 -1
- package/dist/routes/config/channels/endpoints/browse.d.ts.map +0 -1
- package/dist/routes/config/channels/endpoints/bulk.d.ts.map +0 -1
- package/dist/routes/config/channels/endpoints/crud.d.ts.map +0 -1
- package/dist/routes/config/channels/endpoints/importExport.d.ts.map +0 -1
- package/dist/routes/config/channels/endpoints/predefined.d.ts.map +0 -1
- package/dist/routes/config/channels/endpoints/prefs.d.ts.map +0 -1
- package/dist/routes/config/channels/endpoints/service.d.ts.map +0 -1
- package/dist/routes/config/channels/endpoints/tags.d.ts.map +0 -1
- package/dist/routes/config/channels/healthBridge.d.ts.map +0 -1
- package/dist/routes/config/channels/http/handler.d.ts.map +0 -1
- package/dist/routes/config/channels/http/playlistHint.d.ts.map +0 -1
- package/dist/routes/config/channels/http/serviceWarning.d.ts.map +0 -1
- package/dist/routes/config/channels/index.d.ts.map +0 -1
- package/dist/routes/config/channels/setup.d.ts.map +0 -1
- package/dist/routes/config/channels/table.d.ts.map +0 -1
- package/dist/routes/config/http/envelope.d.ts.map +0 -1
- package/dist/routes/config/index.d.ts.map +0 -1
- package/dist/routes/config/services.d.ts.map +0 -1
- package/dist/routes/config/settings.d.ts.map +0 -1
- package/dist/routes/debug.d.ts.map +0 -1
- package/dist/routes/health.d.ts.map +0 -1
- package/dist/routes/hls.d.ts.map +0 -1
- package/dist/routes/icons.d.ts.map +0 -1
- package/dist/routes/index.d.ts.map +0 -1
- package/dist/routes/logs.d.ts.map +0 -1
- package/dist/routes/mpegts.d.ts.map +0 -1
- package/dist/routes/play.d.ts.map +0 -1
- package/dist/routes/playlist.d.ts.map +0 -1
- package/dist/routes/root/content.d.ts.map +0 -1
- package/dist/routes/root/index.d.ts.map +0 -1
- package/dist/routes/root/scripts/channels.d.ts.map +0 -1
- package/dist/routes/root/scripts/clientEscape.d.ts.map +0 -1
- package/dist/routes/root/scripts/clientUrl.d.ts.map +0 -1
- package/dist/routes/root/scripts/config.d.ts.map +0 -1
- package/dist/routes/root/scripts/index.d.ts.map +0 -1
- package/dist/routes/root/scripts/shared.d.ts.map +0 -1
- package/dist/routes/root/scripts/status.d.ts.map +0 -1
- package/dist/routes/root/scripts/status.handlers.d.ts.map +0 -1
- package/dist/routes/root/styles.d.ts.map +0 -1
- package/dist/routes/services.d.ts.map +0 -1
- package/dist/routes/sse.d.ts.map +0 -1
- package/dist/routes/streams.d.ts.map +0 -1
- package/dist/routes/theme.d.ts.map +0 -1
- package/dist/routes/ui.d.ts.map +0 -1
- package/dist/routes/upgrade.d.ts.map +0 -1
- package/dist/service/commands.context.d.ts.map +0 -1
- package/dist/service/commands.d.ts.map +0 -1
- package/dist/service/generators.context.d.ts.map +0 -1
- package/dist/service/generators.d.ts.map +0 -1
- package/dist/service/index.d.ts.map +0 -1
- package/dist/streaming/captureLock.d.ts.map +0 -1
- package/dist/streaming/captureSession.d.ts.map +0 -1
- package/dist/streaming/clients.d.ts.map +0 -1
- package/dist/streaming/codec.d.ts.map +0 -1
- package/dist/streaming/fmp4Segmenter.d.ts.map +0 -1
- package/dist/streaming/hls.d.ts.map +0 -1
- package/dist/streaming/hlsResume.d.ts.map +0 -1
- package/dist/streaming/hlsSegments.d.ts.map +0 -1
- package/dist/streaming/lifecycle.d.ts.map +0 -1
- package/dist/streaming/monitor.d.ts.map +0 -1
- package/dist/streaming/mp4Parser.d.ts.map +0 -1
- package/dist/streaming/mpegts.d.ts.map +0 -1
- package/dist/streaming/playlistBuilder.d.ts.map +0 -1
- package/dist/streaming/preroll.d.ts.map +0 -1
- package/dist/streaming/pretune.d.ts.map +0 -1
- package/dist/streaming/pretuneTimers.d.ts.map +0 -1
- package/dist/streaming/recovery.d.ts.map +0 -1
- package/dist/streaming/registry.d.ts.map +0 -1
- package/dist/streaming/setup.d.ts.map +0 -1
- package/dist/streaming/showInfo.d.ts.map +0 -1
- package/dist/streaming/statusEmitter.d.ts.map +0 -1
- package/dist/types/channels.d.ts.map +0 -1
- package/dist/types/config.d.ts.map +0 -1
- package/dist/types/index.d.ts.map +0 -1
- package/dist/types/profiles.d.ts.map +0 -1
- package/dist/types/selection.d.ts.map +0 -1
- package/dist/types/shared.d.ts.map +0 -1
- package/dist/types/streaming.d.ts.map +0 -1
- package/dist/upgrade/commands.context.d.ts.map +0 -1
- package/dist/upgrade/commands.d.ts.map +0 -1
- package/dist/upgrade/detection.context.d.ts.map +0 -1
- package/dist/upgrade/detection.d.ts.map +0 -1
- package/dist/upgrade/index.d.ts.map +0 -1
- package/dist/upgrade/lifecycle.context.d.ts.map +0 -1
- package/dist/upgrade/lifecycle.d.ts.map +0 -1
- package/dist/upgrade/pathHandle.d.ts.map +0 -1
- package/dist/utils/bootSession.context.d.ts.map +0 -1
- package/dist/utils/bootSession.d.ts.map +0 -1
- package/dist/utils/chromeFetch.d.ts.map +0 -1
- package/dist/utils/cliOutput.d.ts.map +0 -1
- package/dist/utils/clock.d.ts +0 -15
- package/dist/utils/clock.d.ts.map +0 -1
- package/dist/utils/clock.js +0 -31
- package/dist/utils/clock.js.map +0 -1
- package/dist/utils/debugFilter.d.ts.map +0 -1
- package/dist/utils/delay.d.ts.map +0 -1
- package/dist/utils/errors.d.ts.map +0 -1
- package/dist/utils/evaluate.d.ts.map +0 -1
- package/dist/utils/ffmpeg.context.d.ts.map +0 -1
- package/dist/utils/ffmpeg.d.ts.map +0 -1
- package/dist/utils/fileLogger.d.ts.map +0 -1
- package/dist/utils/format.d.ts.map +0 -1
- package/dist/utils/index.d.ts.map +0 -1
- package/dist/utils/logEmitter.d.ts.map +0 -1
- package/dist/utils/logger.d.ts.map +0 -1
- package/dist/utils/m3u.d.ts.map +0 -1
- package/dist/utils/markup.d.ts.map +0 -1
- package/dist/utils/memo.d.ts.map +0 -1
- package/dist/utils/morganStream.d.ts.map +0 -1
- package/dist/utils/network.d.ts.map +0 -1
- package/dist/utils/pid.d.ts.map +0 -1
- package/dist/utils/platform.d.ts.map +0 -1
- package/dist/utils/processInspector.context.d.ts.map +0 -1
- package/dist/utils/processInspector.d.ts.map +0 -1
- package/dist/utils/retry.d.ts.map +0 -1
- package/dist/utils/runtimeIdentity.context.d.ts.map +0 -1
- package/dist/utils/runtimeIdentity.d.ts.map +0 -1
- package/dist/utils/sanitize.d.ts.map +0 -1
- package/dist/utils/streamContext.d.ts.map +0 -1
- package/dist/utils/timing.d.ts.map +0 -1
- package/dist/utils/version.d.ts.map +0 -1
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import { EvaluateTimeoutError, LOG, capitalize, formatError, getAbortSignal, isPageDeathError, isSessionClosedError, runWithStreamContext, startTimer } from "../utils/index.js";
|
|
2
|
-
import { RECOVERY_METHODS, checkCircuitBreaker, classifyNativeSegmentHealth, computeNextRecoveryLevel, createRecoveryMetrics, deriveStreamHealth, formatIssueType, formatRecoveryDuration, getIssueCategory, getIssueDescription, getRecoveryMethod, recordRecoveryAttempt, recordRecoverySuccess, resetCircuitBreaker, shouldTriggerRecovery } from "./recovery.js";
|
|
1
|
+
import { EvaluateTimeoutError, LOG, capitalize, formatError, formatResolution, getAbortSignal, isPageDeathError, isSessionClosedError, runWithStreamContext, startTimer } from "../utils/index.js";
|
|
2
|
+
import { RECOVERY_METHODS, checkCircuitBreaker, classifyNativeSegmentHealth, computeNextRecoveryLevel, createRecoveryMetrics, deriveStreamHealth, describeResolutionOutcome, formatIssueType, formatRecoveryDuration, getIssueCategory, getIssueDescription, getRecoveryMethod, isResolutionDegraded, nextNativeIssueRecord, recordRecoveryAttempt, recordRecoverySuccess, resetCircuitBreaker, resolutionAreaRatio, shouldTriggerRecovery, updateResolutionPeak } from "./recovery.js";
|
|
3
|
+
import { applyNativeQualityRefresh, getLastSegmentHasVideo, getLastSegmentSize, getStream, getStreamMemoryUsage, getStreamSegmenter, isCaptureIdentity, isHardwareAccelerated, makePendingCaptureIdentity } from "./registry.js";
|
|
3
4
|
import { applyVideoStyles, buildVideoSelectorType, checkVideoPresence, enforceVideoVolume, ensurePlayback, findVideoContext, getVideoState, tuneToChannel, validateVideoElement, verifyFullscreen } from "../browser/video.js";
|
|
4
|
-
import {
|
|
5
|
+
import { getCaptureImpairment, syncWindowVisibility } from "../browser/index.js";
|
|
6
|
+
import { getEffectiveCaptureCodec, isCaptureHardwareAccelerated } from "./codec.js";
|
|
5
7
|
import { CONFIG } from "../config/index.js";
|
|
6
8
|
import { clearNativeInitState } from "./hlsSegments.js";
|
|
7
9
|
import { clearProbeCache } from "../native/probe.js";
|
|
@@ -11,8 +13,10 @@ import { getClientSummary } from "./clients.js";
|
|
|
11
13
|
import { getPresetViewport } from "../config/presets.js";
|
|
12
14
|
import { getProviderBySlug } from "../browser/channelSelection.js";
|
|
13
15
|
import { getShowName } from "./showInfo.js";
|
|
14
|
-
import {
|
|
16
|
+
import { reaffirmCaptureSurface } from "../browser/cdp.js";
|
|
15
17
|
import { refreshNativeManifest } from "../native/index.js";
|
|
18
|
+
import { systemClock } from "homebridge-plugin-utils";
|
|
19
|
+
const defaultMonitorDeps = { clock: systemClock, getCaptureImpairment, getEffectiveCaptureCodec, isCaptureHardwareAccelerated, syncWindowVisibility };
|
|
16
20
|
/**
|
|
17
21
|
* Monitors video playback health and attempts escalating recovery when issues are detected. This function runs on an interval, checking video state and triggering
|
|
18
22
|
* increasingly aggressive recovery actions when playback stalls, pauses, or errors occur.
|
|
@@ -36,9 +40,11 @@ import { refreshNativeManifest } from "../native/index.js";
|
|
|
36
40
|
* @param onTabReplacement - Optional callback for tab replacement recovery. When provided and 3+ consecutive timeouts occur, this is called to replace the hung tab.
|
|
37
41
|
* If null/undefined, tab replacement is not available; sustained evaluate timeouts are only surfaced via status (lastIssueType) with no
|
|
38
42
|
* automatic recovery escalation.
|
|
43
|
+
* @param deps - The injected browser-boundary collaborators; defaults to defaultMonitorDeps. Threaded so a test drives the ladder's availability decision and the
|
|
44
|
+
* window sync without a live Chrome.
|
|
39
45
|
* @returns A MonitorHandle exposing the live recovery metrics (getMetrics) and a self-contained dispose that stops the monitor's polling interval.
|
|
40
46
|
*/
|
|
41
|
-
export function monitorPlaybackHealth(page, context, profile, url, streamId, streamInfo, onCircuitBreak, onTabReplacement) {
|
|
47
|
+
export function monitorPlaybackHealth(page, context, profile, url, streamId, streamInfo, onCircuitBreak, onTabReplacement, deps = defaultMonitorDeps) {
|
|
42
48
|
/* Monitor state. These track the video's behavior over time and control recovery decisions. Mutable variables are organized into typed state objects by subsystem
|
|
43
49
|
* (recovery, segments, native health, resolution) to clarify ownership and interaction boundaries. Variables that don't belong to a specific subsystem remain as
|
|
44
50
|
* standalone declarations.
|
|
@@ -46,6 +52,12 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
46
52
|
// The current page reference. This can change after tab replacement recovery, when the old tab is closed and a new one is created. We use a mutable variable so we
|
|
47
53
|
// can update the reference after replacement.
|
|
48
54
|
let currentPage = page;
|
|
55
|
+
/* How many ticks pass between capture-surface re-affirmations, and the count of ticks since this stream's monitor started. Thirty ticks is about a minute at the
|
|
56
|
+
* default cadence, which is the resolution the catch-all is aiming for: fast enough that no disturbance survives long in a recording, slow enough that the
|
|
57
|
+
* command costs nothing measurable next to the health reads the same tick performs.
|
|
58
|
+
*/
|
|
59
|
+
const SURFACE_REAFFIRM_TICK_INTERVAL = 30;
|
|
60
|
+
let reaffirmTickCounter = 0;
|
|
49
61
|
// The video's currentTime from the previous check. Used to detect whether the video is progressing. Null on first check since we have no previous value.
|
|
50
62
|
let lastTime = null;
|
|
51
63
|
// Timestamp when buffering started, or null if not currently buffering. Used to apply the buffering grace period - we don't trigger recovery for buffering until
|
|
@@ -56,9 +68,6 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
56
68
|
// Counter for consecutive page navigation failures. If navigation fails twice in a row, we fall back to source reload (level 2) instead. This prevents getting
|
|
57
69
|
// stuck in a loop when navigation itself is the problem.
|
|
58
70
|
let consecutiveNavigationFailures = 0;
|
|
59
|
-
// Track whether the browser window needs to be re-minimized after recovery. Recovery actions (especially ensureFullscreen) can cause the window to un-minimize.
|
|
60
|
-
// We set this flag when recovery is triggered and clear it after the recovery grace period passes and we see a healthy check.
|
|
61
|
-
let pendingReMinimize = false;
|
|
62
71
|
// Graduated fullscreen reinforcement counter. Counts consecutive ticks where verifyFullscreen() returns false. On tick 1 we apply basic CSS styles (sufficient
|
|
63
72
|
// for well-behaved sites like Hulu). On tick 2+ we escalate to !important priority to override sites that actively fight style changes. Reset to 0 when the
|
|
64
73
|
// video fills the viewport again.
|
|
@@ -125,13 +134,16 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
125
134
|
// appear healthy. This catches the case where Chrome's MediaRecorder silently stops emitting data without raising an error - the input stream stays "open" but no
|
|
126
135
|
// data events fire. The 20-second threshold is 4x the maximum expected moof delivery interval (5 seconds) to avoid false positives during normal bursty delivery.
|
|
127
136
|
const SEGMENT_STALENESS_TIMEOUT = 20000;
|
|
128
|
-
// The capture surface
|
|
129
|
-
// stream is running,
|
|
137
|
+
// The capture surface: the size capture encodes at, reported beside the source's own size in the status the monitor emits. Read once for the monitor's lifetime,
|
|
138
|
+
// because the quality preset is restart-gated and cannot change while a stream is running, so re-deriving it on every two-second tick would be work that can only
|
|
139
|
+
// ever produce the same answer.
|
|
130
140
|
const presetViewport = getPresetViewport(CONFIG);
|
|
131
|
-
// Resolution degradation detection. When the video element's intrinsic resolution
|
|
132
|
-
// low
|
|
133
|
-
//
|
|
134
|
-
|
|
141
|
+
// Resolution degradation detection. When the video element's intrinsic resolution falls well below the best the stream has delivered, the service's ABR is stuck
|
|
142
|
+
// on a low rendition. The threshold is a fraction of that best reading by pixel area. The field's stuck renditions read 16 to 27 percent of their peaks, while
|
|
143
|
+
// an ordinary one-rung adaptive downshift - 1600x900 to 1024x576 on CNN, 41 percent by area - is a service pacing its own bitrate and heals on its own. A third
|
|
144
|
+
// sits between the two: every stuck rendition on record fails it, a one-rung dip passes it, and a source whose best is 720p is never judged against a surface
|
|
145
|
+
// it cannot fill. Each recovery this detector drives is a capture restart, so the line is drawn where the picture has actually collapsed, not merely shrunk.
|
|
146
|
+
const RESOLUTION_RATIO_THRESHOLD = 1 / 3;
|
|
135
147
|
// Grace period in milliseconds after stream start and after each recovery action. Gives ABR time to ramp up before flagging degradation.
|
|
136
148
|
const RESOLUTION_GRACE_PERIOD = 30000;
|
|
137
149
|
// Number of consecutive degraded readings required before triggering recovery. At ~2 seconds per monitor tick, 15 readings = ~30 seconds of sustained
|
|
@@ -141,12 +153,12 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
141
153
|
// navigation and video reinitialization to complete before the site enforces its cutoff.
|
|
142
154
|
const PROACTIVE_RELOAD_MARGIN_MS = 120000;
|
|
143
155
|
// Timestamp of the most recent full page navigation. Used to calculate elapsed continuous playback for proactive reload when maxContinuousPlayback is configured.
|
|
144
|
-
// Initialized to
|
|
156
|
+
// Initialized to the clock's reading because the monitor starts immediately after stream setup establishes playback, meaning a page load just completed. Reset
|
|
145
157
|
// after any successful page navigation recovery or tab replacement, but NOT after source reloads (L2) which preserve the page's JavaScript context.
|
|
146
|
-
let lastPageNavigationTime =
|
|
158
|
+
let lastPageNavigationTime = deps.clock.now();
|
|
147
159
|
// Pre-compute the selector type string for video element selection. This is passed to evaluate() calls.
|
|
148
160
|
const selectorType = buildVideoSelectorType(profile);
|
|
149
|
-
// Capture stream context for re-establishing on each interval tick. AsyncLocalStorage context is lost when entering
|
|
161
|
+
// Capture stream context for re-establishing on each interval tick. AsyncLocalStorage context is lost when entering the callback timer. The show name
|
|
150
162
|
// resolver is lazy - it reads from the live show name cache at log time, so messages always reflect the current program even as shows change mid-stream.
|
|
151
163
|
const streamContext = {
|
|
152
164
|
channelName: streamInfo.channelName ?? undefined,
|
|
@@ -165,13 +177,21 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
165
177
|
stallCount: 0,
|
|
166
178
|
totalAttempts: 0
|
|
167
179
|
};
|
|
180
|
+
/* Whether the ladder can offer tab replacement at this moment. A replacement starts a fresh capture, which a browser marked as unable to start captures refuses,
|
|
181
|
+
* and the handler disposes the working pipeline before it asks - so on a marked browser the ladder treats replacement as unavailable and every decline site keeps
|
|
182
|
+
* the behavior it already has for a stream that was given no handler. The mark is read on each decision rather than captured once, because it can land at any
|
|
183
|
+
* point in a stream's life.
|
|
184
|
+
*/
|
|
185
|
+
const canReplaceTab = () => (onTabReplacement !== undefined) && (deps.getCaptureImpairment() === null);
|
|
186
|
+
// The reason a replacement cannot proceed, put into words in one place so every message that has to explain a decline says the same thing.
|
|
187
|
+
const tabReplacementUnavailableReason = () => (onTabReplacement ? "the browser can no longer start captures" : "no tab replacement handler");
|
|
168
188
|
// Segment production state. Tracks both post-recovery segment verification and continuous segment size monitoring. After L2/L3 recovery, we verify segments are
|
|
169
189
|
// actually being produced. Independently, we monitor segment sizes on every tick to detect spontaneous capture pipeline death (dead pipelines produce tiny segments
|
|
170
190
|
// while the video element appears healthy).
|
|
171
191
|
const segmentState = {
|
|
172
192
|
consecutiveTinySegments: 0,
|
|
173
193
|
lastCheckedIndex: 0,
|
|
174
|
-
lastSegmentAdvanceTime:
|
|
194
|
+
lastSegmentAdvanceTime: deps.clock.now(),
|
|
175
195
|
preRecoveryIndex: null,
|
|
176
196
|
productionStalled: false,
|
|
177
197
|
waitStartTime: null,
|
|
@@ -184,14 +204,15 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
184
204
|
issueTime: null,
|
|
185
205
|
issueType: null,
|
|
186
206
|
lastCheckedSegmentIndex: 0,
|
|
187
|
-
lastSegmentAdvanceTime:
|
|
207
|
+
lastSegmentAdvanceTime: deps.clock.now(),
|
|
188
208
|
recoveryAttempts: 0
|
|
189
209
|
};
|
|
190
210
|
// Resolution degradation monitoring. Separate from the recovery escalation (L1-L3 plus tab replacement) which handles broken playback. Resolution degradation is a
|
|
191
|
-
// quality issue - the stream works but at lower
|
|
211
|
+
// quality issue - the stream works but at lower than the best it has delivered. Uses its own tracking and two-step escalation: page reload, then tab replacement.
|
|
192
212
|
const resolutionState = {
|
|
193
213
|
consecutiveDegradedReadings: 0,
|
|
194
|
-
graceEnd:
|
|
214
|
+
graceEnd: deps.clock.now() + RESOLUTION_GRACE_PERIOD,
|
|
215
|
+
peak: null,
|
|
195
216
|
recoveryAttempt: 0
|
|
196
217
|
};
|
|
197
218
|
/**
|
|
@@ -208,12 +229,15 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
208
229
|
* @param entry - The stream registry entry for the native stream.
|
|
209
230
|
*/
|
|
210
231
|
function checkNativeStreamHealth(entry) {
|
|
211
|
-
|
|
212
|
-
|
|
232
|
+
// The narrow happens here rather than being inherited from the tick's dispatch: a narrowing does not cross a function call, and the entry's identity can be
|
|
233
|
+
// replaced by a fallback that ran between the dispatch and this frame. A stream that is no longer native has nothing here to check.
|
|
234
|
+
const identity = entry.identity;
|
|
235
|
+
if (identity.mode !== "native") {
|
|
213
236
|
emitStatusUpdate();
|
|
214
237
|
return;
|
|
215
238
|
}
|
|
216
|
-
const
|
|
239
|
+
const proxy = identity.nativeProxy;
|
|
240
|
+
const now = deps.clock.now();
|
|
217
241
|
const currentSegmentIndex = proxy.getSegmentIndex();
|
|
218
242
|
const lastSegmentTime = proxy.getLastSegmentTime();
|
|
219
243
|
const targetDuration = proxy.getTargetDuration();
|
|
@@ -222,9 +246,12 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
222
246
|
// Fast path: if the proxy hit its error threshold and stopped itself, trigger L3 fallback immediately. This avoids waiting for the staleness threshold when hard
|
|
223
247
|
// errors (HTTP 403, network failures) have already been detected by the proxy's internal retry loop.
|
|
224
248
|
if (proxy.hasErrored()) {
|
|
225
|
-
LOG.debug("native:monitor", "Native proxy errored for %s. Initiating capture fallback.", storeKey);
|
|
226
249
|
nativeHealthState.issueType = "proxy error";
|
|
227
250
|
nativeHealthState.issueTime = now;
|
|
251
|
+
if (deferFallbackInsideGrace(entry, identity)) {
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
LOG.debug("native:monitor", "Native proxy errored for %s. Initiating capture fallback.", storeKey);
|
|
228
255
|
// The caller establishes the stream context for this interval tick, so the fire-and-forget recovery promise inherits it across its async continuations.
|
|
229
256
|
void executeNativeL3Fallback(entry);
|
|
230
257
|
return;
|
|
@@ -252,11 +279,12 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
252
279
|
stalenessMs,
|
|
253
280
|
targetDurationMs: targetDuration * 1000
|
|
254
281
|
});
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
282
|
+
/* Advance the recorded issue through the pure rule in recovery.ts: an unchanged cause keeps its original start time across ticks, and a changed cause takes
|
|
283
|
+
* both the new label and a fresh stamp. A stream that degrades from fetch errors into a segment stall therefore reports the stall it is actually in.
|
|
284
|
+
*/
|
|
285
|
+
const issueRecord = nextNativeIssueRecord(nativeHealthState, decision.issueType, now);
|
|
286
|
+
nativeHealthState.issueTime = issueRecord.issueTime;
|
|
287
|
+
nativeHealthState.issueType = issueRecord.issueType;
|
|
260
288
|
const staleSec = Math.round(stalenessMs / 1000);
|
|
261
289
|
// Diagnostic logging mirrors the classification. The staleness threshold reported is 2x the target duration in seconds.
|
|
262
290
|
if (decision.health === "recovering") {
|
|
@@ -276,42 +304,69 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
276
304
|
}
|
|
277
305
|
// L3: at 4x target duration once an L2 attempt has already been made and the stall persists, fall back to capture mode via tab replacement.
|
|
278
306
|
if (decision.action === "l3") {
|
|
307
|
+
if (deferFallbackInsideGrace(entry, identity)) {
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
279
310
|
LOG.warn("Falling back to capture mode for %s: native streaming stalled after recovery attempt.", storeKey);
|
|
280
311
|
// The caller establishes the stream context for this interval tick, so the fire-and-forget recovery promise inherits it across its async continuations.
|
|
281
312
|
void executeNativeL3Fallback(entry);
|
|
282
313
|
return;
|
|
283
314
|
}
|
|
284
|
-
emitNativeStatus(entry, decision.health);
|
|
315
|
+
emitNativeStatus(entry, identity, decision.health);
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* Holds a native capture fallback at the decision site while the recovery grace window is open, reporting the stream as recovering instead of entering the
|
|
319
|
+
* fallback cycle. Every native trigger - a relay that stopped itself on its error threshold, a stall the ladder has escalated to its last rung - describes a
|
|
320
|
+
* condition that holds on every tick once it is true. Without this read the whole cycle - the mode pre-flip, the attempt, the revert, the window sync, the
|
|
321
|
+
* warning - would run twice a second inside a window every other trigger is respecting. Nothing is lost by waiting, since neither condition is curing itself in
|
|
322
|
+
* the meantime, and the status still goes out so the display keeps advancing. The replacement primitive's own entry gate stays the guarantee beneath this: a
|
|
323
|
+
* gate here can be forgotten, that one cannot be bypassed.
|
|
324
|
+
*
|
|
325
|
+
* The status is recovering for every trigger, because a fallback being held is a recovery in progress from the display's point of view.
|
|
326
|
+
*
|
|
327
|
+
* @param entry - The stream registry entry.
|
|
328
|
+
* @param identity - The stream's native identity, narrowed by the caller.
|
|
329
|
+
* @returns True when the fallback was held, which is the caller's signal to return without entering it.
|
|
330
|
+
*/
|
|
331
|
+
function deferFallbackInsideGrace(entry, identity) {
|
|
332
|
+
if (!isWithinRecoveryGrace()) {
|
|
333
|
+
return false;
|
|
334
|
+
}
|
|
335
|
+
LOG.debug("native:monitor", "Capture fallback for %s waits for the recovery grace window to close.", entry.info.storeKey);
|
|
336
|
+
emitNativeStatus(entry, identity, "recovering");
|
|
337
|
+
return true;
|
|
285
338
|
}
|
|
286
339
|
/**
|
|
287
340
|
* Emits a status update with native-specific health classification. Populates meaningful fields (health, issue tracking, memory, clients) and zeroes video-specific
|
|
288
341
|
* fields that are not applicable to native streams.
|
|
289
342
|
*
|
|
290
343
|
* @param entry - The stream registry entry.
|
|
344
|
+
* @param identity - The stream's native identity, narrowed by the caller so the manifest quality members are read without a second mode test.
|
|
291
345
|
* @param health - The health status to report.
|
|
292
346
|
*/
|
|
293
|
-
function emitNativeStatus(entry, health) {
|
|
347
|
+
function emitNativeStatus(entry, identity, health) {
|
|
294
348
|
if (intervalCleared) {
|
|
295
349
|
return;
|
|
296
350
|
}
|
|
297
|
-
const now =
|
|
351
|
+
const now = deps.clock.now();
|
|
298
352
|
const memoryBytes = getStreamMemoryUsage(entry).total;
|
|
299
353
|
const channelKey = entry.info.storeKey;
|
|
300
|
-
const clientSummary = getClientSummary(streamInfo.numericStreamId);
|
|
354
|
+
const clientSummary = getClientSummary(streamInfo.numericStreamId, now);
|
|
301
355
|
// Native streams have no recovery ladder, so we reuse escalationLevel purely as a UI severity encoding rather than as a recovery-level index: healthy maps to 0,
|
|
302
356
|
// stalled to 1, and recovering to 2. The intended ordering is stalled < recovering because an actively-recovering stream warrants a stronger visual signal than one
|
|
303
357
|
// that has merely stalled.
|
|
304
358
|
const escalation = (health === "stalled") ? 1 : ((health === "recovering") ? 2 : 0);
|
|
305
359
|
const status = {
|
|
306
360
|
bufferingDuration: null,
|
|
307
|
-
captureCodec:
|
|
361
|
+
captureCodec: identity.captureCodec,
|
|
362
|
+
captureResolution: null,
|
|
308
363
|
channel: streamInfo.channelName,
|
|
309
364
|
clientCount: clientSummary.total,
|
|
310
365
|
clients: clientSummary.clients,
|
|
311
366
|
currentTime: 0,
|
|
312
|
-
duration: Math.round((now - streamInfo.startTime
|
|
367
|
+
duration: Math.round((now - streamInfo.startTime) / 1000),
|
|
313
368
|
escalationLevel: escalation,
|
|
314
|
-
hardwareAccelerated: entry
|
|
369
|
+
hardwareAccelerated: isHardwareAccelerated(entry),
|
|
315
370
|
health,
|
|
316
371
|
id: streamInfo.numericStreamId,
|
|
317
372
|
lastIssueTime: nativeHealthState.issueTime,
|
|
@@ -319,16 +374,17 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
319
374
|
lastRecoveryTime: null,
|
|
320
375
|
logoUrl: channelKey ? (getChannelLogo(channelKey) ?? "") : "",
|
|
321
376
|
memoryBytes,
|
|
322
|
-
nativeBandwidth:
|
|
323
|
-
nativeResolution:
|
|
377
|
+
nativeBandwidth: identity.nativeBandwidth,
|
|
378
|
+
nativeResolution: identity.nativeResolution,
|
|
324
379
|
networkState: 0,
|
|
325
380
|
pageReloadsInWindow: 0,
|
|
326
381
|
readyState: 0,
|
|
327
382
|
recoveryAttempts: nativeHealthState.recoveryAttempts,
|
|
328
383
|
serviceName: streamInfo.serviceName,
|
|
329
384
|
showName: getShowName(streamInfo.numericStreamId),
|
|
330
|
-
|
|
331
|
-
|
|
385
|
+
sourceResolution: null,
|
|
386
|
+
startTime: new Date(streamInfo.startTime).toISOString(),
|
|
387
|
+
streamingMode: identity.mode,
|
|
332
388
|
url
|
|
333
389
|
};
|
|
334
390
|
emitStreamHealthChanged(status);
|
|
@@ -340,39 +396,35 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
340
396
|
* @param entry - The stream registry entry.
|
|
341
397
|
*/
|
|
342
398
|
async function executeNativeL2Recovery(entry) {
|
|
343
|
-
|
|
344
|
-
|
|
399
|
+
// The narrow is taken fresh here, not inherited from the caller: this runs as a fire-and-forget continuation, and a stream that has since fallen back to
|
|
400
|
+
// capture has no manifest left to refresh.
|
|
401
|
+
const identity = entry.identity;
|
|
402
|
+
if ((identity.mode !== "native") || identity.nativeProxy.isStopped()) {
|
|
345
403
|
return;
|
|
346
404
|
}
|
|
405
|
+
const proxy = identity.nativeProxy;
|
|
347
406
|
/* The refresh probes under the identity the tune established, which the entry holds. It is null only for a pending entry whose setup has not filled it in,
|
|
348
407
|
* and a stream with a running native proxy is past that point - so this answers a window that recovery does not reach rather than papering over one.
|
|
349
|
-
* Declining is the
|
|
408
|
+
* Declining is the answer either way: an identity assembled here would stamp a binding this frame cannot see, and the stall escalates to L3.
|
|
350
409
|
*/
|
|
351
410
|
const probeIdentity = entry.probeIdentity;
|
|
352
411
|
if (!probeIdentity) {
|
|
353
412
|
LOG.debug("native:monitor", "Skipping L2 recovery for %s: the stream has no probe-cache identity yet.", entry.info.storeKey);
|
|
354
413
|
return;
|
|
355
414
|
}
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
*/
|
|
360
|
-
const reestablishManifest = entry.reestablishManifest;
|
|
361
|
-
if (!reestablishManifest) {
|
|
362
|
-
LOG.debug("native:monitor", "Skipping L2 recovery for %s: the stream has no re-establishment capability.", entry.info.storeKey);
|
|
363
|
-
return;
|
|
364
|
-
}
|
|
415
|
+
// The refresh's page-reload strategy re-establishes the channel through the capability the stream's setup built. Every write that produces a native identity
|
|
416
|
+
// supplies it, which the type states, so this frame reads it rather than testing for it.
|
|
417
|
+
const reestablishManifest = identity.reestablishManifest;
|
|
365
418
|
recoveryState.inProgress = true;
|
|
366
419
|
LOG.debug("native:monitor", "Starting L2 recovery (page reload) for %s.", entry.info.storeKey);
|
|
367
420
|
try {
|
|
368
421
|
const success = await refreshNativeManifest({
|
|
369
422
|
channelName: entry.info.storeKey,
|
|
423
|
+
clock: deps.clock,
|
|
370
424
|
onFeedApplied: (metadata) => {
|
|
371
425
|
// The registry write belongs to this layer, not to the native one: recovery already holds the entry, so a refresh that binds a different rung of the
|
|
372
426
|
// service's ladder is recorded here as the stream's current quality.
|
|
373
|
-
entry
|
|
374
|
-
entry.nativeBandwidth = metadata.bandwidth;
|
|
375
|
-
entry.nativeResolution = metadata.resolution;
|
|
427
|
+
applyNativeQualityRefresh(entry, metadata);
|
|
376
428
|
},
|
|
377
429
|
page: currentPage,
|
|
378
430
|
probeIdentity,
|
|
@@ -383,7 +435,7 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
383
435
|
});
|
|
384
436
|
if (success) {
|
|
385
437
|
// Reset staleness tracking so the monitor gives the refreshed stream time to produce segments.
|
|
386
|
-
nativeHealthState.lastSegmentAdvanceTime =
|
|
438
|
+
nativeHealthState.lastSegmentAdvanceTime = deps.clock.now();
|
|
387
439
|
}
|
|
388
440
|
}
|
|
389
441
|
finally {
|
|
@@ -394,68 +446,127 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
394
446
|
* L3 recovery for native streams: falls back to capture mode via tab replacement. Stops the native proxy, creates a fresh page with capture pipeline, and switches
|
|
395
447
|
* the stream to capture mode. The existing tab replacement infrastructure handles page creation, capture initialization, segmenter creation, and registry updates.
|
|
396
448
|
*
|
|
449
|
+
* The relay keeps producing throughout the attempt and is stopped only on the exits that hand the stream to something else, so a fallback that cannot establish
|
|
450
|
+
* its capture leaves the stream exactly as it found it rather than stranded between two modes.
|
|
451
|
+
*
|
|
397
452
|
* If the proxy's onError fires concurrently (from the poll loop hitting its failure threshold), terminateStream runs before this async function gets a chance to
|
|
398
|
-
* execute. By the time L3 runs, the stream is already terminated and executeTabReplacement
|
|
453
|
+
* execute. By the time L3 runs, the stream is already terminated and executeTabReplacement reports the stop.
|
|
399
454
|
*
|
|
400
455
|
* @param entry - The stream registry entry.
|
|
401
456
|
*/
|
|
402
457
|
async function executeNativeL3Fallback(entry) {
|
|
403
|
-
|
|
404
|
-
|
|
458
|
+
// The whole identity this fallback is leaving, held in one local. The narrow is taken fresh here because a narrowing does not cross a call, and the failed
|
|
459
|
+
// arm restores this same object rather than reconstructing it member by member.
|
|
460
|
+
const previousIdentity = entry.identity;
|
|
461
|
+
if (previousIdentity.mode !== "native") {
|
|
462
|
+
return;
|
|
463
|
+
}
|
|
464
|
+
if (!canReplaceTab()) {
|
|
465
|
+
LOG.warn("Capture fallback not available for %s: %s.", entry.info.storeKey, tabReplacementUnavailableReason());
|
|
405
466
|
onCircuitBreak();
|
|
406
467
|
return;
|
|
407
468
|
}
|
|
408
469
|
LOG.debug("native:monitor", "Starting L3 fallback (capture mode) for %s.", entry.info.storeKey);
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
470
|
+
/* Declare capture mode before the replacement rather than after it. The window-visibility policy reads the registry, and createPageWithCapture syncs the window
|
|
471
|
+
* at the top of establishment - so an entry still marked native there would let the policy minimize the window under a capture that is being acquired. Every
|
|
472
|
+
* outcome below either keeps this or reverts it, so the entry never rests on a mode it is not in.
|
|
473
|
+
*
|
|
474
|
+
* The proxy is deliberately left running behind that declaration, right through the attempt. The replacement builds its capture on a new page and does not
|
|
475
|
+
* touch the old one until the swap, so there is nothing for a polling proxy to trip over, and a fallback that fails hands back a stream that never stopped
|
|
476
|
+
* relaying. Its token-refresh timer keeps running too: a refresh landing mid-window is ordinary native operation, and the registry's quality write skips it
|
|
477
|
+
* because the entry is holding a capture identity for the duration.
|
|
478
|
+
*/
|
|
479
|
+
entry.identity = makePendingCaptureIdentity();
|
|
414
480
|
// Use the existing tab replacement infrastructure. It sets recoveryState.inProgress = true internally and clears it in finalizeTabReplacement. It creates a new
|
|
415
|
-
// page with capture, navigates, sets up playback, creates a segmenter, and
|
|
416
|
-
// attached to it).
|
|
481
|
+
// page with capture, navigates, sets up playback, creates a segmenter, and swaps the registry entry over to the new page and capture session.
|
|
417
482
|
const outcome = await executeTabReplacement("native fallback to capture");
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
483
|
+
/* An exhaustive switch rather than a chain, so a future outcome cannot be absorbed silently into a catch-all. No rejection handling wraps the await above:
|
|
484
|
+
* executeTabReplacement converts every throw into an outcome in its own catch, so there is no rejection to arm for.
|
|
485
|
+
*/
|
|
486
|
+
switch (outcome.outcome) {
|
|
487
|
+
case "success": {
|
|
488
|
+
/* The handler set the page and the new capture session (with its segmenter attached) on the entry, and capture mode was declared before the replacement,
|
|
489
|
+
* so what remains is the state the native path left behind.
|
|
490
|
+
*
|
|
491
|
+
* Clear separate audio state from the native proxy. Without this, hasAudio remains true and the HLS handler continues serving the master playlist
|
|
492
|
+
* (referencing video.m3u8 and audio.m3u8) instead of the capture segmenter's variant playlist. Clients that cached the master playlist structure would
|
|
493
|
+
* request stale audio and video variant playlists pointing to segments that are no longer being updated.
|
|
494
|
+
*/
|
|
495
|
+
entry.hls.hasAudio = false;
|
|
496
|
+
entry.hls.audioPlaylist = "";
|
|
497
|
+
entry.hls.audioSegments.clear();
|
|
498
|
+
entry.hls.audioSegmentBytes = 0;
|
|
499
|
+
entry.hls.videoPlaylist = "";
|
|
500
|
+
// Release the relay's initialization segments. They belong to a source this stream no longer consumes, and the memory report reads their byte counter with
|
|
501
|
+
// no mode gate, so state left here would be counted for the rest of the stream's life. The nativeContainer label needs no matching reset: it lives on the
|
|
502
|
+
// native identity this entry no longer holds.
|
|
503
|
+
clearNativeInitState(entry.id);
|
|
504
|
+
// Clear the probe cache so subsequent tunes to this channel don't re-attempt native streaming.
|
|
505
|
+
clearProbeCache(entry.info.storeKey);
|
|
506
|
+
/* Re-derive the codec facts from the capture decision. The identity the handler wrote carried forward what the pre-swap identity held, which on this path
|
|
507
|
+
* is the pending shape the pre-flip installed - and before that, the label read off the service's manifest, which describes a feed this stream is no
|
|
508
|
+
* longer consuming. The status display would otherwise report that stale label for the rest of the stream's life. This is the one sanctioned within-variant
|
|
509
|
+
* spread: the capture session and the page the handler installed are left exactly as they are.
|
|
510
|
+
*/
|
|
511
|
+
if (isCaptureIdentity(entry)) {
|
|
512
|
+
entry.identity = { ...entry.identity, captureCodec: deps.getEffectiveCaptureCodec().toUpperCase(), hardwareAccelerated: deps.isCaptureHardwareAccelerated() };
|
|
513
|
+
}
|
|
514
|
+
LOG.info("Switched to capture mode for %s: native streaming failed.", entry.info.storeKey);
|
|
515
|
+
// The monitor's next tick sees a capture identity and runs the normal video element monitoring path. The state reset from applyTabReplacementSuccess
|
|
516
|
+
// (called by executeTabReplacement) already initialized all capture-mode monitor variables.
|
|
517
|
+
break;
|
|
518
|
+
}
|
|
519
|
+
case "failed": {
|
|
520
|
+
/* The replacement did not take, but the stream continues un-terminated - and because nothing was disposed before the swap, what it continues on is the
|
|
521
|
+
* native relay it has been running all along. Restoring the held identity whole hands back the same live proxy, and the window sync lets the presentation
|
|
522
|
+
* settle now that no capture is being attempted.
|
|
523
|
+
*
|
|
524
|
+
* The restore is correct in both entry conditions. A proxy that is still healthy simply resumes. A proxy that had already stopped itself on its error
|
|
525
|
+
* threshold re-triggers this fallback on a later tick - deliberately: the grace window throttles each cycle and the circuit breaker bounds the count, so a
|
|
526
|
+
* native stream whose relay died and whose capture fallback keeps failing escalates to termination rather than resting in a silent limbo.
|
|
527
|
+
*/
|
|
528
|
+
entry.identity = previousIdentity;
|
|
529
|
+
void deps.syncWindowVisibility();
|
|
530
|
+
LOG.warn("Capture fallback failed for %s: tab replacement unsuccessful.", entry.info.storeKey);
|
|
531
|
+
break;
|
|
532
|
+
}
|
|
533
|
+
case "deferred": {
|
|
534
|
+
/* The replacement was declined before it started because the previous recovery's window is still open. Nothing was attempted, so the revert is the same one
|
|
535
|
+
* the failed arm performs and the narration is a breadcrumb rather than a warning.
|
|
536
|
+
*
|
|
537
|
+
* No live native path reaches this arm: every native trigger holds its fallback at the decision site while the window is open, and nothing runs between
|
|
538
|
+
* that read and the primitive's own that could open a window. The arm stays because the outcome union is shared with the capture triggers and the switch
|
|
539
|
+
* is exhaustive by design.
|
|
540
|
+
*/
|
|
541
|
+
entry.identity = previousIdentity;
|
|
542
|
+
void deps.syncWindowVisibility();
|
|
543
|
+
LOG.debug("native:monitor", "Capture fallback for %s deferred inside the recovery grace window.", entry.info.storeKey);
|
|
544
|
+
break;
|
|
545
|
+
}
|
|
546
|
+
case "stopped": {
|
|
547
|
+
// The monitor stopped while the replacement was in flight, which means the stream terminated. Termination disposed what the registry held - the pending
|
|
548
|
+
// capture identity, which holds nothing - so the proxy is this frame's to release.
|
|
549
|
+
break;
|
|
550
|
+
}
|
|
551
|
+
case "terminated": {
|
|
552
|
+
// The circuit breaker tripped during the replacement and terminated the stream synchronously, again while the registry held the pending identity, so
|
|
553
|
+
// nothing else can reach the proxy.
|
|
554
|
+
LOG.warn("Capture fallback failed for %s: circuit breaker tripped.", entry.info.storeKey);
|
|
555
|
+
break;
|
|
556
|
+
}
|
|
449
557
|
}
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
558
|
+
/* One stop site owns the whole leak matrix. The two arms that put the held identity back are the two that hand the proxy on to a stream that is still using
|
|
559
|
+
* it; every other exit is the end of this proxy's life - the capture pipeline has taken over, or the stream is gone and took nothing with it - and an
|
|
560
|
+
* unstopped proxy would go on polling and refreshing forever.
|
|
561
|
+
*/
|
|
562
|
+
if ((outcome.outcome !== "failed") && (outcome.outcome !== "deferred")) {
|
|
563
|
+
previousIdentity.nativeProxy.stop();
|
|
453
564
|
}
|
|
454
565
|
}
|
|
455
566
|
// Helper to mark a discontinuity in the HLS playlist after recovery events that disrupt the video source. The segmenter flushes its current fragment buffer and sets
|
|
456
567
|
// a pending discontinuity flag so the next segment boundary includes an #EXT-X-DISCONTINUITY tag. This tells HLS clients to flush their decoder state.
|
|
457
568
|
const markStreamDiscontinuity = () => {
|
|
458
|
-
getStream(streamInfo.numericStreamId)?.
|
|
569
|
+
getStreamSegmenter(getStream(streamInfo.numericStreamId))?.markDiscontinuity();
|
|
459
570
|
};
|
|
460
571
|
/**
|
|
461
572
|
* Computes the health status classification based on current monitor state.
|
|
@@ -480,24 +591,33 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
480
591
|
if (intervalCleared) {
|
|
481
592
|
return;
|
|
482
593
|
}
|
|
483
|
-
const now =
|
|
594
|
+
const now = deps.clock.now();
|
|
484
595
|
// Get current memory usage from the stream's HLS segment buffers.
|
|
485
596
|
const entry = getStream(streamInfo.numericStreamId);
|
|
486
597
|
const memoryBytes = entry ? getStreamMemoryUsage(entry).total : 0;
|
|
487
598
|
// Get the channel key from the registry entry for logo lookup.
|
|
488
599
|
const channelKey = entry?.info.storeKey ?? "";
|
|
489
600
|
// Get current client counts and type breakdown for this stream.
|
|
490
|
-
const clientSummary = getClientSummary(streamInfo.numericStreamId);
|
|
601
|
+
const clientSummary = getClientSummary(streamInfo.numericStreamId, now);
|
|
602
|
+
// An entry that is gone projects as the capture defaults, which is the same shape the flat wire carries for a stream whose setup has not filled it in yet.
|
|
603
|
+
const identity = entry?.identity;
|
|
604
|
+
const nativeIdentity = (identity?.mode === "native") ? identity : null;
|
|
605
|
+
// The two capture-only sizes are null for a native entry: the entry's mode is the contract, not the builder.
|
|
606
|
+
const captureStream = identity?.mode !== "native";
|
|
607
|
+
const sourceResolution = (captureStream && lastVideoState && (lastVideoState.videoWidth > 0) && (lastVideoState.videoHeight > 0)) ?
|
|
608
|
+
formatResolution(lastVideoState.videoWidth, lastVideoState.videoHeight) : null;
|
|
609
|
+
const captureResolution = captureStream ? formatResolution(presetViewport.width, presetViewport.height) : null;
|
|
491
610
|
const status = {
|
|
492
611
|
bufferingDuration: bufferingStartTime ? Math.round((now - bufferingStartTime) / 1000) : null,
|
|
493
|
-
captureCodec:
|
|
612
|
+
captureCodec: identity?.captureCodec ?? null,
|
|
613
|
+
captureResolution,
|
|
494
614
|
channel: streamInfo.channelName,
|
|
495
615
|
clientCount: clientSummary.total,
|
|
496
616
|
clients: clientSummary.clients,
|
|
497
617
|
currentTime: lastVideoState?.time ?? 0,
|
|
498
|
-
duration: Math.round((now - streamInfo.startTime
|
|
618
|
+
duration: Math.round((now - streamInfo.startTime) / 1000),
|
|
499
619
|
escalationLevel: recoveryState.escalationLevel,
|
|
500
|
-
hardwareAccelerated: entry
|
|
620
|
+
hardwareAccelerated: entry ? isHardwareAccelerated(entry) : false,
|
|
501
621
|
health: computeHealthStatus(),
|
|
502
622
|
id: streamInfo.numericStreamId,
|
|
503
623
|
lastIssueTime,
|
|
@@ -505,16 +625,17 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
505
625
|
lastRecoveryTime: recoveryState.lastRecoveryTime > 0 ? recoveryState.lastRecoveryTime : null,
|
|
506
626
|
logoUrl: channelKey ? (getChannelLogo(channelKey) ?? "") : "",
|
|
507
627
|
memoryBytes,
|
|
508
|
-
nativeBandwidth:
|
|
509
|
-
nativeResolution:
|
|
628
|
+
nativeBandwidth: nativeIdentity?.nativeBandwidth ?? 0,
|
|
629
|
+
nativeResolution: nativeIdentity?.nativeResolution ?? null,
|
|
510
630
|
networkState: lastVideoState?.networkState ?? 0,
|
|
511
631
|
pageReloadsInWindow: pageReloadTimestamps.length,
|
|
512
632
|
readyState: lastVideoState?.readyState ?? 0,
|
|
513
633
|
recoveryAttempts: recoveryState.totalAttempts,
|
|
514
634
|
serviceName: streamInfo.serviceName,
|
|
515
635
|
showName: getShowName(streamInfo.numericStreamId),
|
|
516
|
-
|
|
517
|
-
|
|
636
|
+
sourceResolution,
|
|
637
|
+
startTime: new Date(streamInfo.startTime).toISOString(),
|
|
638
|
+
streamingMode: identity?.mode ?? "capture",
|
|
518
639
|
url
|
|
519
640
|
};
|
|
520
641
|
emitStreamHealthChanged(status);
|
|
@@ -537,8 +658,8 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
537
658
|
segmentState.productionStalled = false;
|
|
538
659
|
segmentState.consecutiveTinySegments = 0;
|
|
539
660
|
segmentState.wasInTinyState = false;
|
|
540
|
-
segmentState.lastCheckedIndex = getStream(streamInfo.numericStreamId)?.
|
|
541
|
-
segmentState.lastSegmentAdvanceTime =
|
|
661
|
+
segmentState.lastCheckedIndex = getStreamSegmenter(getStream(streamInfo.numericStreamId))?.getSegmentIndex() ?? 0;
|
|
662
|
+
segmentState.lastSegmentAdvanceTime = deps.clock.now();
|
|
542
663
|
}
|
|
543
664
|
/**
|
|
544
665
|
* Resets all failure/retry counters to zero. Called after successful tab replacement or page navigation to give the stream a fresh start.
|
|
@@ -552,11 +673,13 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
552
673
|
videoNotFoundCount = 0;
|
|
553
674
|
}
|
|
554
675
|
/**
|
|
555
|
-
* Resets resolution monitoring
|
|
676
|
+
* Resets the resolution monitoring a recovery action makes stale: the degraded-reading count, the grace window, and the ladder step. Deliberately not the peak
|
|
677
|
+
* record - the source's rendition ladder does not change with a reload or a tab replacement, and the record's accepted flag is what keeps an unrelated recovery
|
|
678
|
+
* from re-running the resolution ladder inside one degraded episode.
|
|
556
679
|
*/
|
|
557
680
|
function resetResolutionState() {
|
|
558
681
|
resolutionState.consecutiveDegradedReadings = 0;
|
|
559
|
-
resolutionState.graceEnd =
|
|
682
|
+
resolutionState.graceEnd = deps.clock.now() + RESOLUTION_GRACE_PERIOD;
|
|
560
683
|
resolutionState.recoveryAttempt = 0;
|
|
561
684
|
}
|
|
562
685
|
/**
|
|
@@ -565,12 +688,12 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
565
688
|
* @returns True if a page reload is allowed and the timestamp has been recorded, false if the rate limit has been reached.
|
|
566
689
|
*/
|
|
567
690
|
function isPageReloadAllowed() {
|
|
568
|
-
const reloadWindow =
|
|
691
|
+
const reloadWindow = deps.clock.now() - CONFIG.playback.pageReloadWindow;
|
|
569
692
|
pageReloadTimestamps = pageReloadTimestamps.filter((ts) => ts > reloadWindow);
|
|
570
693
|
if (pageReloadTimestamps.length >= CONFIG.playback.maxPageReloads) {
|
|
571
694
|
return false;
|
|
572
695
|
}
|
|
573
|
-
pageReloadTimestamps.push(
|
|
696
|
+
pageReloadTimestamps.push(deps.clock.now());
|
|
574
697
|
return true;
|
|
575
698
|
}
|
|
576
699
|
/**
|
|
@@ -581,13 +704,22 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
581
704
|
recoveryState.sourceReloadAttempted = false;
|
|
582
705
|
}
|
|
583
706
|
/**
|
|
584
|
-
* Sets the recovery grace period
|
|
585
|
-
*
|
|
707
|
+
* Sets the recovery grace period after a recovery action. The grace period prevents the monitor from immediately detecting new issues while the recovery action
|
|
708
|
+
* takes effect.
|
|
586
709
|
* @param level - The recovery level (1-3) to determine grace period duration.
|
|
587
710
|
*/
|
|
588
711
|
function setRecoveryGracePeriod(level) {
|
|
589
|
-
|
|
590
|
-
|
|
712
|
+
recoveryState.graceUntil = deps.clock.now() + (recoveryGracePeriods[level] ?? 0);
|
|
713
|
+
}
|
|
714
|
+
/**
|
|
715
|
+
* Reports whether the monitor is inside the post-recovery grace window. One state, read here by every consumer that has to honour it: the tick computes the
|
|
716
|
+
* value it threads into the health checks from this, and the triggers that sit outside that thread - the tiny-segment gate, the unresponsive-tab gate, the
|
|
717
|
+
* native fallback triggers through deferFallbackInsideGrace, and the replacement primitive's own entry gate - call it directly. A second way of asking the
|
|
718
|
+
* same question is how a trigger ends up escalating inside a window every other trigger is respecting.
|
|
719
|
+
* @returns True while the grace window from the last recovery action is still open.
|
|
720
|
+
*/
|
|
721
|
+
function isWithinRecoveryGrace() {
|
|
722
|
+
return deps.clock.now() < recoveryState.graceUntil;
|
|
591
723
|
}
|
|
592
724
|
/**
|
|
593
725
|
* Stops the monitoring interval. Pairs the two operations that must always happen together: setting intervalCleared so any in-flight async tick short-circuits at
|
|
@@ -597,7 +729,7 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
597
729
|
*/
|
|
598
730
|
function stopMonitoring() {
|
|
599
731
|
intervalCleared = true;
|
|
600
|
-
|
|
732
|
+
interval[Symbol.dispose]();
|
|
601
733
|
}
|
|
602
734
|
/**
|
|
603
735
|
* Reports whether the monitor stopped while a recovery action was awaiting, and releases the recovery flag when it did. A recovery await can resume after
|
|
@@ -628,7 +760,7 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
628
760
|
if (intervalCleared) {
|
|
629
761
|
return { outcome: "stopped" };
|
|
630
762
|
}
|
|
631
|
-
const cbResult = checkCircuitBreaker(circuitBreaker,
|
|
763
|
+
const cbResult = checkCircuitBreaker(circuitBreaker, deps.clock.now());
|
|
632
764
|
if (cbResult.shouldTrip) {
|
|
633
765
|
LOG.error("Recovery exhausted (%s) - terminating stream.", context);
|
|
634
766
|
stopMonitoring();
|
|
@@ -645,11 +777,12 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
645
777
|
function applyTabReplacementSuccess(result) {
|
|
646
778
|
currentPage = result.page;
|
|
647
779
|
currentContext = result.context;
|
|
648
|
-
const
|
|
780
|
+
const settledAt = deps.clock.now();
|
|
781
|
+
const duration = formatRecoveryDuration(metrics.currentRecoveryStartTime ?? settledAt, settledAt);
|
|
649
782
|
LOG.info("Recovered in %s via %s.", duration, RECOVERY_METHODS.tabReplacement);
|
|
650
|
-
recordRecoverySuccess(metrics, RECOVERY_METHODS.tabReplacement);
|
|
783
|
+
recordRecoverySuccess(metrics, RECOVERY_METHODS.tabReplacement, settledAt);
|
|
651
784
|
// Full state reset for fresh tab.
|
|
652
|
-
lastPageNavigationTime =
|
|
785
|
+
lastPageNavigationTime = deps.clock.now();
|
|
653
786
|
resetRecoveryCounters();
|
|
654
787
|
resetEscalationState();
|
|
655
788
|
resetSegmentMonitoringState();
|
|
@@ -659,7 +792,8 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
659
792
|
}
|
|
660
793
|
/**
|
|
661
794
|
* Handles tab replacement failure after all retry attempts are exhausted. Clears stale recovery metrics (preventing ghost "Recovered" logs from the
|
|
662
|
-
* deferred-success check),
|
|
795
|
+
* deferred-success check), arms the grace window that throttles the next attempt while one can still start, runs the circuit breaker, and catches the one state
|
|
796
|
+
* that still strands a stream.
|
|
663
797
|
* @param context - Description of the failure for circuit breaker logging.
|
|
664
798
|
* @returns The tab replacement outcome (failed or terminated).
|
|
665
799
|
*/
|
|
@@ -673,11 +807,27 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
673
807
|
// Clear stale recovery metrics so the deferred-success check does not falsely log "Recovered" from leftover state set by recordRecoveryAttempt.
|
|
674
808
|
metrics.currentRecoveryStartTime = null;
|
|
675
809
|
metrics.currentRecoveryMethod = null;
|
|
676
|
-
LOG.warn("Tab replacement unsuccessful
|
|
810
|
+
LOG.warn("Tab replacement was unsuccessful. The stream continues on its existing capture.");
|
|
811
|
+
/* Arm the grace window on the way out, the same window a success arms. A failed replacement leaves the stream running on its old pipeline, still exhibiting
|
|
812
|
+
* whatever condition triggered the attempt - so without this the very next tick re-satisfies that condition and fires again, and a persistently degraded
|
|
813
|
+
* stream spends its life in a replacement loop. With it, the cadence is one real attempt per window (ten seconds at this level, from the monitor's grace
|
|
814
|
+
* table), each true failure counted by the circuit breaker, ending in termination when the breaker's threshold trips. Bounded escalation rather than a tight
|
|
815
|
+
* loop, and no log line per suppressed tick.
|
|
816
|
+
*
|
|
817
|
+
* The arming reads whether a replacement can still start, because on a browser that can start none there is no replacement loop here to throttle: every
|
|
818
|
+
* trigger reaches its no-replacement arm rather than the handler. Holding the arms that terminate for a window holds every other recording's re-tune behind
|
|
819
|
+
* this stream, because the relaunch that cures the browser waits on the last stream's end. A failure that itself marked the browser lands the mark before the
|
|
820
|
+
* exhaustion settles, so the read here sees it.
|
|
821
|
+
*/
|
|
822
|
+
if (canReplaceTab()) {
|
|
823
|
+
setRecoveryGracePeriod(3);
|
|
824
|
+
}
|
|
677
825
|
const failureOutcome = handleTabReplacementFailure(context);
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
826
|
+
/* The safety net for the one state that still strands a stream: termination raced the exhaustion and took the page while this settled. A replacement that
|
|
827
|
+
* simply failed leaves the old page open and serving, so this does not fire for it - that stream continues on its existing capture with the breaker counting,
|
|
828
|
+
* which is the designed outcome. When the page really is gone, the next tick would silently clear the interval via currentPage.isClosed() with no termination
|
|
829
|
+
* log, no status emission, and no cleanup, leaving a zombie entry in the registry. Terminate explicitly instead.
|
|
830
|
+
*/
|
|
681
831
|
if ((failureOutcome.outcome === "failed") && currentPage.isClosed()) {
|
|
682
832
|
LOG.error("Tab replacement failed and the original page is no longer available - terminating stream.");
|
|
683
833
|
stopMonitoring();
|
|
@@ -688,27 +838,38 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
688
838
|
}
|
|
689
839
|
/**
|
|
690
840
|
* Executes tab replacement recovery with full error handling. This unified helper handles all tab replacement triggers (tiny segments, stalled capture, unresponsive
|
|
691
|
-
* tab) consistently, including metrics recording, success/failure logging, circuit breaker checks, and state resets.
|
|
841
|
+
* tab, resolution degradation, native fallback) consistently, including metrics recording, success/failure logging, circuit breaker checks, and state resets.
|
|
842
|
+
*
|
|
843
|
+
* The handler builds the replacement capture before disposing anything: it creates a fresh page, acquires its capture, and tunes it while the existing pipeline
|
|
844
|
+
* keeps producing, and only then swaps in one synchronous frame. So a failed attempt costs the stream nothing - the old capture is still running and still
|
|
845
|
+
* serving the recording - and the single retry below is a second chance rather than the last one. The retry re-reads continuity from the still-live old
|
|
846
|
+
* segmenter, and its own fresh resources are discarded on failure the same way the first attempt's were.
|
|
692
847
|
*
|
|
693
|
-
*
|
|
694
|
-
* createPageWithCapture, so a retry is the only chance to save the stream when the first attempt fails. All handler cleanup steps are safe to call more than
|
|
695
|
-
* once on retry: rawCaptureStream.destroyed guard, segmenter stop() checks state.stopped, FFmpeg kill() checks ffmpeg.killed, page close checks
|
|
696
|
-
* !oldPage.isClosed(), and unregisterManagedPage is a no-op on repeat.
|
|
848
|
+
* Every entry runs through the grace check below, which is what keeps a persistently degraded stream from spending its life re-attempting.
|
|
697
849
|
* @param issueType - Description of what triggered the replacement (for logging and UI display).
|
|
698
850
|
* @returns The tab replacement outcome.
|
|
699
851
|
*/
|
|
700
852
|
async function executeTabReplacement(issueType) {
|
|
701
|
-
// Guard: caller
|
|
702
|
-
if (!onTabReplacement) {
|
|
853
|
+
// Guard: the caller has already consulted canReplaceTab, but TypeScript needs the explicit narrowing for the awaited calls below.
|
|
854
|
+
if (!onTabReplacement || !canReplaceTab()) {
|
|
703
855
|
return { outcome: "failed" };
|
|
704
856
|
}
|
|
857
|
+
/* The throttle, enforced at the one place every trigger passes through. Individual triggers consult the grace window in their own gates so their warnings and
|
|
858
|
+
* side effects stay quiet too, but a gate can be forgotten and a new trigger can be added, whereas this cannot be bypassed by construction. It sits ahead of
|
|
859
|
+
* every piece of bookkeeping - before the attempt metric, before the recovery flags, before the handler runs - so a deferral costs nothing and is
|
|
860
|
+
* indistinguishable from the attempt never having been asked for.
|
|
861
|
+
*/
|
|
862
|
+
if (isWithinRecoveryGrace()) {
|
|
863
|
+
LOG.debug("recovery:tab", "Deferring the %s tab replacement: the previous recovery's grace window is still open.", issueType);
|
|
864
|
+
return { outcome: "deferred" };
|
|
865
|
+
}
|
|
705
866
|
recoveryState.inProgress = true;
|
|
706
867
|
recoveryState.totalAttempts++;
|
|
707
|
-
recoveryState.lastRecoveryTime =
|
|
868
|
+
recoveryState.lastRecoveryTime = deps.clock.now();
|
|
708
869
|
lastIssueType = issueType;
|
|
709
|
-
lastIssueTime =
|
|
710
|
-
const tabRecoveryElapsed = startTimer();
|
|
711
|
-
recordRecoveryAttempt(metrics, RECOVERY_METHODS.tabReplacement);
|
|
870
|
+
lastIssueTime = deps.clock.now();
|
|
871
|
+
const tabRecoveryElapsed = startTimer(deps.clock);
|
|
872
|
+
recordRecoveryAttempt(metrics, RECOVERY_METHODS.tabReplacement, deps.clock.now());
|
|
712
873
|
try {
|
|
713
874
|
let result = await onTabReplacement();
|
|
714
875
|
// First attempt failed - retry once. See the safe-to-retry notes in the JSDoc above.
|
|
@@ -735,7 +896,7 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
735
896
|
}
|
|
736
897
|
catch (error) {
|
|
737
898
|
// Unexpected error (not from onTabReplacement - those are caught internally by the handler in hls.ts and return null). Guard against registry corruption,
|
|
738
|
-
//
|
|
899
|
+
// capture-acquisition failures, or other unexpected errors.
|
|
739
900
|
LOG.debug("recovery:tab", "Tab replacement attempt 1/2 failed: %s. Retrying...", formatError(error));
|
|
740
901
|
try {
|
|
741
902
|
const retryResult = await onTabReplacement();
|
|
@@ -769,14 +930,14 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
769
930
|
* @returns Recovery result with the new context if successful.
|
|
770
931
|
*/
|
|
771
932
|
async function performPageNavigationRecovery() {
|
|
772
|
-
const navRecoveryElapsed = startTimer();
|
|
933
|
+
const navRecoveryElapsed = startTimer(deps.clock);
|
|
773
934
|
// Track page count before navigation to detect unexpected new tabs (popups, ads).
|
|
774
935
|
const browser = currentPage.browser();
|
|
775
936
|
const pageCountBefore = (await browser.pages()).length;
|
|
776
937
|
try {
|
|
777
938
|
// Use tuneToChannel to reinitialize playback. It is the tune this recovery path owns, and it runs the phases stream setup ran - navigation, channel
|
|
778
939
|
// selection, video detection, click-to-play, playback - so a recovered stream comes up the way a fresh one did.
|
|
779
|
-
const { context: newContext } = await tuneToChannel(currentPage, url, profile);
|
|
940
|
+
const { context: newContext } = await tuneToChannel(currentPage, url, profile, deps.clock);
|
|
780
941
|
// Check for unexpected new tabs created during tuning.
|
|
781
942
|
const pageCountAfter = (await browser.pages()).length;
|
|
782
943
|
if (pageCountAfter > pageCountBefore) {
|
|
@@ -848,7 +1009,7 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
848
1009
|
if (videoNotFoundCount === 2) {
|
|
849
1010
|
LOG.debug("recovery:context", "Re-searching frames for video element.");
|
|
850
1011
|
try {
|
|
851
|
-
const newContext = await findVideoContext(currentPage, profile);
|
|
1012
|
+
const newContext = await findVideoContext(currentPage, profile, deps.clock);
|
|
852
1013
|
const validationState = await validateVideoElement(newContext, selectorType);
|
|
853
1014
|
if (validationState.found) {
|
|
854
1015
|
LOG.info("Video found in different context after re-search. readyState=%s.", validationState.readyState);
|
|
@@ -889,9 +1050,8 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
889
1050
|
recoveryState.escalationLevel = 3;
|
|
890
1051
|
recoveryState.lastRecoveryTime = now;
|
|
891
1052
|
recoveryState.totalAttempts++;
|
|
892
|
-
pendingReMinimize = true;
|
|
893
1053
|
recoveryState.inProgress = true;
|
|
894
|
-
recordRecoveryAttempt(metrics, RECOVERY_METHODS.pageNavigation);
|
|
1054
|
+
recordRecoveryAttempt(metrics, RECOVERY_METHODS.pageNavigation, deps.clock.now());
|
|
895
1055
|
// Check page reload limit before attempting recovery.
|
|
896
1056
|
if (!isPageReloadAllowed()) {
|
|
897
1057
|
LOG.error("Page navigation rate limit reached (%s in %s minutes) - cannot recover without video element.", CONFIG.playback.maxPageReloads, Math.round(CONFIG.playback.pageReloadWindow / 60000));
|
|
@@ -912,10 +1072,11 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
912
1072
|
setRecoveryGracePeriod(3);
|
|
913
1073
|
if (recoveryResult.success && recoveryResult.newContext) {
|
|
914
1074
|
currentContext = recoveryResult.newContext;
|
|
915
|
-
const
|
|
1075
|
+
const settledAt = deps.clock.now();
|
|
1076
|
+
const duration = formatRecoveryDuration(metrics.currentRecoveryStartTime ?? settledAt, settledAt);
|
|
916
1077
|
LOG.info("Recovered in %s via %s.", duration, RECOVERY_METHODS.pageNavigation);
|
|
917
|
-
recordRecoverySuccess(metrics, RECOVERY_METHODS.pageNavigation);
|
|
918
|
-
lastPageNavigationTime =
|
|
1078
|
+
recordRecoverySuccess(metrics, RECOVERY_METHODS.pageNavigation, settledAt);
|
|
1079
|
+
lastPageNavigationTime = deps.clock.now();
|
|
919
1080
|
resetRecoveryCounters();
|
|
920
1081
|
resetEscalationState();
|
|
921
1082
|
resetSegmentMonitoringState();
|
|
@@ -948,7 +1109,7 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
948
1109
|
segmentState.waitStartTime ??= now;
|
|
949
1110
|
// Check if segments are flowing by comparing current index to pre-recovery index.
|
|
950
1111
|
const entry = getStream(streamInfo.numericStreamId);
|
|
951
|
-
const currentIndex = entry?.
|
|
1112
|
+
const currentIndex = getStreamSegmenter(entry)?.getSegmentIndex() ?? null;
|
|
952
1113
|
if ((currentIndex !== null) && (currentIndex > segmentState.preRecoveryIndex)) {
|
|
953
1114
|
// Segments are flowing - recovery actually succeeded. Clear tracking state.
|
|
954
1115
|
segmentState.preRecoveryIndex = null;
|
|
@@ -963,7 +1124,7 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
963
1124
|
}
|
|
964
1125
|
// Continuous segment size monitoring. Runs on every healthy interval to detect spontaneous capture pipeline death.
|
|
965
1126
|
const sizeCheckEntry = getStream(streamInfo.numericStreamId);
|
|
966
|
-
const currentSegmentIndex = sizeCheckEntry?.
|
|
1127
|
+
const currentSegmentIndex = getStreamSegmenter(sizeCheckEntry)?.getSegmentIndex() ?? 0;
|
|
967
1128
|
if ((currentSegmentIndex > segmentState.lastCheckedIndex) && sizeCheckEntry) {
|
|
968
1129
|
// A new segment was produced. Update the staleness tracker and check its size.
|
|
969
1130
|
segmentState.lastSegmentAdvanceTime = now;
|
|
@@ -977,14 +1138,21 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
977
1138
|
const hasVideo = getLastSegmentHasVideo(sizeCheckEntry);
|
|
978
1139
|
const effectiveThreshold = (hasVideo === false) ? TINY_SEGMENT_COUNT_TRIGGER : providerTinySegmentThreshold;
|
|
979
1140
|
LOG.debug("recovery:tracks", "Below-threshold segment: %d bytes, hasVideo=%s, consecutive=%d, threshold=%d.", segmentSize, String(hasVideo), segmentState.consecutiveTinySegments, effectiveThreshold);
|
|
980
|
-
|
|
1141
|
+
/* Act on sustained undersized segments, but not inside the post-recovery grace window. The grace term is what keeps a stream that a replacement did not
|
|
1142
|
+
* cure from re-escalating every two seconds: a fresh capture needs its window to start producing real segments, and a stream still emitting tiny ones
|
|
1143
|
+
* when the window closes will satisfy this again immediately. Suppressing the whole branch - the warning and the circuit-breaker fallback alongside the
|
|
1144
|
+
* replacement - is deliberate, and mirrors what the staleness branch below has always done with its own grace term. The counter keeps advancing
|
|
1145
|
+
* meanwhile, so the evidence is not lost, only the reaction is deferred.
|
|
1146
|
+
*/
|
|
1147
|
+
if ((segmentState.consecutiveTinySegments >= effectiveThreshold) && !withinRecoveryGrace) {
|
|
981
1148
|
LOG.warn("Detected %d consecutive undersized segments (%dKB) - capture pipeline may have stalled.", segmentState.consecutiveTinySegments, Math.round(segmentSize / 1024));
|
|
982
|
-
//
|
|
983
|
-
|
|
1149
|
+
// Replace the tab when a replacement can start, and otherwise raise the stalled-production flag. The recovery action reads that flag on this same tick
|
|
1150
|
+
// and terminates the stream, because no replacement can be started and the in-page ladder cannot revive a dead capture.
|
|
1151
|
+
if (canReplaceTab() && !recoveryState.inProgress) {
|
|
984
1152
|
await executeTabReplacement("tiny segments");
|
|
985
1153
|
return true;
|
|
986
1154
|
}
|
|
987
|
-
else if (!
|
|
1155
|
+
else if (!canReplaceTab()) {
|
|
988
1156
|
segmentState.productionStalled = true;
|
|
989
1157
|
}
|
|
990
1158
|
}
|
|
@@ -1012,20 +1180,21 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1012
1180
|
* after tab replacement while the new capture pipeline is initializing).
|
|
1013
1181
|
*/
|
|
1014
1182
|
LOG.warn("No new segments produced for %ss - capture pipeline may have stalled.", SEGMENT_STALENESS_TIMEOUT / 1000);
|
|
1015
|
-
if (
|
|
1183
|
+
if (canReplaceTab() && !recoveryState.inProgress) {
|
|
1016
1184
|
await executeTabReplacement("segment staleness");
|
|
1017
1185
|
return true;
|
|
1018
1186
|
}
|
|
1019
|
-
else if (!
|
|
1187
|
+
else if (!canReplaceTab()) {
|
|
1020
1188
|
segmentState.productionStalled = true;
|
|
1021
1189
|
}
|
|
1022
1190
|
}
|
|
1023
1191
|
return false;
|
|
1024
1192
|
}
|
|
1025
1193
|
/**
|
|
1026
|
-
* Monitors video resolution against the
|
|
1027
|
-
* restart), then tab replacement (fresh page with new network connections).
|
|
1028
|
-
*
|
|
1194
|
+
* Monitors video resolution against the best resolution the stream has delivered and triggers recovery for sustained ABR degradation. Uses a two-step escalation:
|
|
1195
|
+
* page reload (forces ABR restart), then tab replacement (fresh page with new network connections). Acceptance after both attempts is recorded on the peak record
|
|
1196
|
+
* and ends with the episode, so the ladder runs at most once per level the source has demonstrated rather than looping on content that is simply low-resolution.
|
|
1197
|
+
* Also detects and logs resolution restoration after successful recovery.
|
|
1029
1198
|
*
|
|
1030
1199
|
* @param now - Current timestamp for timing calculations.
|
|
1031
1200
|
* @param state - Current video state with intrinsic resolution.
|
|
@@ -1038,12 +1207,34 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1038
1207
|
if (!isProgressing || state.paused || state.error || state.ended || withinRecoveryGrace || (state.videoWidth === 0) || (state.videoHeight === 0)) {
|
|
1039
1208
|
return false;
|
|
1040
1209
|
}
|
|
1041
|
-
const
|
|
1042
|
-
const
|
|
1043
|
-
|
|
1044
|
-
|
|
1210
|
+
const reading = { height: state.videoHeight, width: state.videoWidth };
|
|
1211
|
+
const peak = updateResolutionPeak({ peak: resolutionState.peak, reading });
|
|
1212
|
+
resolutionState.peak = peak;
|
|
1213
|
+
const isDegraded = isResolutionDegraded({ peak, reading, threshold: RESOLUTION_RATIO_THRESHOLD });
|
|
1214
|
+
/* The picture is back at or near its peak: the episode is over, whatever the ladder had reached. Say so once, then clear every trace of the episode - the
|
|
1215
|
+
* accepted flag included - so a later drop starts a fresh ladder. The method the message names is read from the ladder step, which an unrelated recovery may
|
|
1216
|
+
* have zeroed while the acceptance stood, so the sentence stays truthful in every case.
|
|
1217
|
+
*/
|
|
1218
|
+
if (!isDegraded) {
|
|
1219
|
+
if (peak.accepted || (resolutionState.recoveryAttempt > 0)) {
|
|
1220
|
+
const method = (resolutionState.recoveryAttempt === 1) ? "page reload" :
|
|
1221
|
+
((resolutionState.recoveryAttempt === 2) ? "tab replacement" : "an unrelated recovery");
|
|
1222
|
+
const verb = describeResolutionOutcome({ peak, reading });
|
|
1223
|
+
LOG.info("Video resolution %s to %s\u00d7%s after %s.", verb, String(state.videoWidth), String(state.videoHeight), method);
|
|
1224
|
+
resolutionState.consecutiveDegradedReadings = 0;
|
|
1225
|
+
resolutionState.graceEnd = 0;
|
|
1226
|
+
resolutionState.peak = { ...peak, accepted: false };
|
|
1227
|
+
resolutionState.recoveryAttempt = 0;
|
|
1228
|
+
}
|
|
1229
|
+
return false;
|
|
1230
|
+
}
|
|
1231
|
+
// A ladder that already ran to acceptance at this peak does not run again inside the same episode: the source has shown nothing better since.
|
|
1232
|
+
if (peak.accepted) {
|
|
1233
|
+
return false;
|
|
1234
|
+
}
|
|
1235
|
+
if (now >= resolutionState.graceEnd) {
|
|
1045
1236
|
resolutionState.consecutiveDegradedReadings++;
|
|
1046
|
-
LOG.debug("recovery:resolution", "Video resolution: %s\u00d7%s (
|
|
1237
|
+
LOG.debug("recovery:resolution", "Video resolution: %s\u00d7%s (peak: %s\u00d7%s, area: %s%%, consecutive: %s/%s).", String(state.videoWidth), String(state.videoHeight), String(peak.width), String(peak.height), String(Math.round(100 * resolutionAreaRatio({ peak, reading }))), String(resolutionState.consecutiveDegradedReadings), String(RESOLUTION_DEGRADED_COUNT_THRESHOLD));
|
|
1047
1238
|
}
|
|
1048
1239
|
else {
|
|
1049
1240
|
resolutionState.consecutiveDegradedReadings = 0;
|
|
@@ -1052,7 +1243,7 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1052
1243
|
// (RESOLUTION_DEGRADED_COUNT_THRESHOLD consecutive readings) to let transient ABR dips self-heal.
|
|
1053
1244
|
if ((resolutionState.consecutiveDegradedReadings >= RESOLUTION_DEGRADED_COUNT_THRESHOLD) && (resolutionState.recoveryAttempt === 0)) {
|
|
1054
1245
|
const degradedDuration = resolutionState.consecutiveDegradedReadings * 2;
|
|
1055
|
-
LOG.warn("Video resolution has been degraded for %ss (%s\u00d7%s
|
|
1246
|
+
LOG.warn("Video resolution has been degraded for %ss (%s\u00d7%s against a %s\u00d7%s peak). Attempting recovery via %s.", String(degradedDuration), String(state.videoWidth), String(state.videoHeight), String(peak.width), String(peak.height), RECOVERY_METHODS.pageNavigation);
|
|
1056
1247
|
recoveryState.inProgress = true;
|
|
1057
1248
|
if (!isPageReloadAllowed()) {
|
|
1058
1249
|
LOG.warn("Resolution recovery deferred - page navigation rate limit reached (%s in %s minutes).", CONFIG.playback.maxPageReloads, Math.round(CONFIG.playback.pageReloadWindow / 60000));
|
|
@@ -1062,7 +1253,6 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1062
1253
|
emitStatusUpdate();
|
|
1063
1254
|
return true;
|
|
1064
1255
|
}
|
|
1065
|
-
pendingReMinimize = true;
|
|
1066
1256
|
const recoveryResult = await performPageNavigationRecovery();
|
|
1067
1257
|
// Same post-await stop check as the other navigation sites: a resumption after the monitor stopped applies neither the grace windows nor the ratchet step
|
|
1068
1258
|
// below, since both would be bookkeeping for a stream that is gone.
|
|
@@ -1080,7 +1270,7 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1080
1270
|
resolutionState.graceEnd = now + RESOLUTION_GRACE_PERIOD;
|
|
1081
1271
|
if (recoveryResult.success && recoveryResult.newContext) {
|
|
1082
1272
|
currentContext = recoveryResult.newContext;
|
|
1083
|
-
lastPageNavigationTime =
|
|
1273
|
+
lastPageNavigationTime = deps.clock.now();
|
|
1084
1274
|
// Reset the general-recovery state a successful reload made stale, composed for this site. Deliberately NOT resetResolutionState() (which the proactive-reload
|
|
1085
1275
|
// sibling calls): that zeroes the recoveryAttempt ratchet set to 1 just above, collapsing the two-step resolution ladder into an endless step-1 loop. And
|
|
1086
1276
|
// deliberately WITH resetEscalationState() (which the sibling omits): the sibling's caller guarantees a zero escalation level, but this path can run mid-cooldown
|
|
@@ -1098,7 +1288,7 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1098
1288
|
}
|
|
1099
1289
|
// Escalation step 2: tab replacement. Creates a fresh page with new capture pipeline and network connections.
|
|
1100
1290
|
if ((resolutionState.consecutiveDegradedReadings >= RESOLUTION_DEGRADED_COUNT_THRESHOLD) && (resolutionState.recoveryAttempt === 1)) {
|
|
1101
|
-
if (
|
|
1291
|
+
if (canReplaceTab()) {
|
|
1102
1292
|
const degradedDuration = resolutionState.consecutiveDegradedReadings * 2;
|
|
1103
1293
|
LOG.warn("Video resolution is still degraded after %ss (%s\u00d7%s). Attempting recovery via %s.", String(degradedDuration), String(state.videoWidth), String(state.videoHeight), RECOVERY_METHODS.tabReplacement);
|
|
1104
1294
|
await executeTabReplacement("resolution degraded");
|
|
@@ -1110,21 +1300,13 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1110
1300
|
// Tab replacement not available - skip directly to acceptance.
|
|
1111
1301
|
resolutionState.recoveryAttempt = 2;
|
|
1112
1302
|
}
|
|
1113
|
-
|
|
1303
|
+
/* Acceptance: still degraded after both recovery attempts. Log once and record the acceptance on the peak record, which is what holds the ladder for the rest
|
|
1304
|
+
* of this episode. The ladder step stays at 2 so the eventual restoration names the tab replacement, and the reading count is zeroed so no further step fires.
|
|
1305
|
+
*/
|
|
1114
1306
|
if ((resolutionState.consecutiveDegradedReadings >= RESOLUTION_DEGRADED_COUNT_THRESHOLD) && (resolutionState.recoveryAttempt === 2)) {
|
|
1115
|
-
LOG.warn("Video resolution remains degraded (%s\u00d7%s
|
|
1116
|
-
resolutionState.recoveryAttempt = 3;
|
|
1117
|
-
}
|
|
1118
|
-
// Resolution is good: clear tracking state so future degradation starts fresh. Use "restored" when resolution matches the viewport, "improved" when it's
|
|
1119
|
-
// above the degradation threshold but below the viewport. Include the recovery method so this single message tells the complete story.
|
|
1120
|
-
if (!isDegraded && (resolutionState.recoveryAttempt > 0)) {
|
|
1121
|
-
const isFullQuality = (state.videoWidth >= presetViewport.width) && (state.videoHeight >= presetViewport.height);
|
|
1122
|
-
const verb = isFullQuality ? "restored" : "improved";
|
|
1123
|
-
const method = (resolutionState.recoveryAttempt === 1) ? "page reload" : "tab replacement";
|
|
1124
|
-
LOG.info("Video resolution %s to %s\u00d7%s after %s.", verb, String(state.videoWidth), String(state.videoHeight), method);
|
|
1307
|
+
LOG.warn("Video resolution remains degraded (%s\u00d7%s against a %s\u00d7%s peak) after recovery attempts. The stream will continue at reduced quality.", String(state.videoWidth), String(state.videoHeight), String(peak.width), String(peak.height));
|
|
1125
1308
|
resolutionState.consecutiveDegradedReadings = 0;
|
|
1126
|
-
resolutionState.
|
|
1127
|
-
resolutionState.graceEnd = 0;
|
|
1309
|
+
resolutionState.peak = { ...peak, accepted: true };
|
|
1128
1310
|
}
|
|
1129
1311
|
return false;
|
|
1130
1312
|
}
|
|
@@ -1158,10 +1340,9 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1158
1340
|
// yield. If the site eventually cuts the stream, normal error recovery handles it.
|
|
1159
1341
|
if (!isPageReloadAllowed()) {
|
|
1160
1342
|
LOG.warn("Proactive reload deferred - page navigation rate limit reached (%s in %s minutes).", CONFIG.playback.maxPageReloads, Math.round(CONFIG.playback.pageReloadWindow / 60000));
|
|
1161
|
-
// Set a grace period
|
|
1162
|
-
//
|
|
1163
|
-
|
|
1164
|
-
recoveryState.graceUntil = now + recoveryGracePeriods[3];
|
|
1343
|
+
// Set a grace period so this deferral does not re-trigger every 2 seconds while the rate limit remains in effect. Level 3 is the bound a page navigation
|
|
1344
|
+
// would have taken, which is the action being deferred.
|
|
1345
|
+
setRecoveryGracePeriod(3);
|
|
1165
1346
|
recoveryState.inProgress = false;
|
|
1166
1347
|
emitStatusUpdate();
|
|
1167
1348
|
return true;
|
|
@@ -1176,7 +1357,7 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1176
1357
|
setRecoveryGracePeriod(3);
|
|
1177
1358
|
if (recoveryResult.success && recoveryResult.newContext) {
|
|
1178
1359
|
currentContext = recoveryResult.newContext;
|
|
1179
|
-
lastPageNavigationTime =
|
|
1360
|
+
lastPageNavigationTime = deps.clock.now();
|
|
1180
1361
|
LOG.info("Proactive reload completed successfully.");
|
|
1181
1362
|
resetRecoveryCounters();
|
|
1182
1363
|
resetSegmentMonitoringState();
|
|
@@ -1205,17 +1386,26 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1205
1386
|
* @returns True if the tick should exit (circuit breaker tripped or tab replacement triggered), false to continue.
|
|
1206
1387
|
*/
|
|
1207
1388
|
async function executeRecoveryAction(now, state, isProgressing, isBuffering) {
|
|
1208
|
-
//
|
|
1209
|
-
//
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
await executeTabReplacement("capture pipeline stalled");
|
|
1389
|
+
// Post-await stop check, covering every exit below it. The tick awaited getVideoState before calling this; a tick that resumed after the monitor stopped must
|
|
1390
|
+
// trip nothing and attempt nothing, because the accounting, the log, the replacement, and the termination would all describe a stream that has already ended.
|
|
1391
|
+
// Return terminal so the tick exits without further work.
|
|
1392
|
+
if (intervalCleared) {
|
|
1213
1393
|
return true;
|
|
1214
1394
|
}
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1395
|
+
/* Segment production stall handling. The capture pipeline is dead and the in-page ladder cannot revive a capture, so a replacement is the one cure. When none
|
|
1396
|
+
* can start, the stream is unrecoverable: every rung the ladder would run spends an attempt and part of the breaker's window on a stream it cannot save, while
|
|
1397
|
+
* holding open the relaunch that would cure a marked browser. So the stream terminates through the breaker at once, which is the judgment the unresponsive-tab
|
|
1398
|
+
* branch already makes.
|
|
1399
|
+
*/
|
|
1400
|
+
if (segmentState.productionStalled) {
|
|
1401
|
+
if (canReplaceTab()) {
|
|
1402
|
+
LOG.warn("Capture pipeline still stalled - escalating to %s.", RECOVERY_METHODS.tabReplacement);
|
|
1403
|
+
await executeTabReplacement("capture pipeline stalled");
|
|
1404
|
+
return true;
|
|
1405
|
+
}
|
|
1406
|
+
LOG.error("Capture pipeline stalled and tab replacement is unavailable (%s) - terminating stream.", tabReplacementUnavailableReason());
|
|
1407
|
+
stopMonitoring();
|
|
1408
|
+
onCircuitBreak();
|
|
1219
1409
|
return true;
|
|
1220
1410
|
}
|
|
1221
1411
|
// Check circuit breaker for too many failures.
|
|
@@ -1239,7 +1429,6 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1239
1429
|
recoveryState.escalationLevel = nextLevel;
|
|
1240
1430
|
recoveryState.lastRecoveryTime = now;
|
|
1241
1431
|
recoveryState.totalAttempts++;
|
|
1242
|
-
pendingReMinimize = true;
|
|
1243
1432
|
// Get recovery method name for logging and metrics.
|
|
1244
1433
|
const recoveryMethod = getRecoveryMethod(recoveryState.escalationLevel);
|
|
1245
1434
|
// Store issue type and time for UI display.
|
|
@@ -1254,19 +1443,18 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1254
1443
|
const issueDesc = getIssueDescription(issueCategory);
|
|
1255
1444
|
LOG.warn("Playback %s - recovering via %s.", issueDesc, recoveryMethod);
|
|
1256
1445
|
}
|
|
1257
|
-
recordRecoveryAttempt(metrics, recoveryMethod);
|
|
1446
|
+
recordRecoveryAttempt(metrics, recoveryMethod, deps.clock.now());
|
|
1258
1447
|
// For L2/L3 recovery, record the current segment index so we can verify segments are flowing after recovery completes.
|
|
1259
1448
|
if (recoveryState.escalationLevel >= 2) {
|
|
1260
1449
|
const entry = getStream(streamInfo.numericStreamId);
|
|
1261
|
-
segmentState.preRecoveryIndex = entry?.
|
|
1450
|
+
segmentState.preRecoveryIndex = getStreamSegmenter(entry)?.getSegmentIndex() ?? null;
|
|
1262
1451
|
segmentState.waitStartTime = null;
|
|
1263
|
-
segmentState.productionStalled = false;
|
|
1264
1452
|
}
|
|
1265
1453
|
recoveryState.inProgress = true;
|
|
1266
1454
|
try {
|
|
1267
1455
|
// Levels 1-2: In-page recovery via ensurePlayback().
|
|
1268
1456
|
if (recoveryState.escalationLevel <= 2) {
|
|
1269
|
-
await ensurePlayback(currentPage, currentContext, profile, { recoveryLevel: recoveryState.escalationLevel, skipNativeFullscreen: true });
|
|
1457
|
+
await ensurePlayback(currentPage, currentContext, profile, { clock: deps.clock, recoveryLevel: recoveryState.escalationLevel, skipNativeFullscreen: true });
|
|
1270
1458
|
// The stream can terminate while ensurePlayback runs. A resumption after the monitor stopped applies none of the settlement below - the source-reload
|
|
1271
1459
|
// mark, the discontinuity, the grace window, the resolution reset. Returning true is what makes the caller exit the tick immediately.
|
|
1272
1460
|
if (abandonRecoveryIfStopped()) {
|
|
@@ -1305,10 +1493,11 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1305
1493
|
setRecoveryGracePeriod(3);
|
|
1306
1494
|
if (recoveryResult.success && recoveryResult.newContext) {
|
|
1307
1495
|
currentContext = recoveryResult.newContext;
|
|
1308
|
-
const
|
|
1496
|
+
const settledAt = deps.clock.now();
|
|
1497
|
+
const duration = formatRecoveryDuration(metrics.currentRecoveryStartTime ?? settledAt, settledAt);
|
|
1309
1498
|
LOG.info("Recovered in %s via %s.", duration, RECOVERY_METHODS.pageNavigation);
|
|
1310
|
-
recordRecoverySuccess(metrics, RECOVERY_METHODS.pageNavigation);
|
|
1311
|
-
lastPageNavigationTime =
|
|
1499
|
+
recordRecoverySuccess(metrics, RECOVERY_METHODS.pageNavigation, settledAt);
|
|
1500
|
+
lastPageNavigationTime = deps.clock.now();
|
|
1312
1501
|
resetRecoveryCounters();
|
|
1313
1502
|
resetEscalationState();
|
|
1314
1503
|
resetSegmentMonitoringState();
|
|
@@ -1357,11 +1546,11 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1357
1546
|
* 2. tickInProgress, so a firing that lands while the previous body is still running skips instead of stacking a second body on the same counters.
|
|
1358
1547
|
* 3. recoveryState.inProgress. On the capture path this covers the window a recovery await holds open inside a tick body; on the native path it is the only
|
|
1359
1548
|
* guard there is, since a native recovery is launched fire-and-forget and outlives the tick that started it.
|
|
1360
|
-
* 4. currentPage.isClosed(), for page termination outside of recovery. It must come after the recovery check:
|
|
1361
|
-
*
|
|
1362
|
-
* permanently.
|
|
1549
|
+
* 4. currentPage.isClosed(), for page termination outside of recovery. It must come after the recovery check: the page reference this monitor holds is the one
|
|
1550
|
+
* a replacement is about to retire, and the swap closes it, so a tick landing between the close and applyTabReplacementSuccess's re-point would read the
|
|
1551
|
+
* closed page and clear the interval mid-recovery, stopping status updates permanently.
|
|
1363
1552
|
*/
|
|
1364
|
-
const interval =
|
|
1553
|
+
const interval = deps.clock.schedule(() => {
|
|
1365
1554
|
// Stop monitoring if cleanup was requested.
|
|
1366
1555
|
if (intervalCleared) {
|
|
1367
1556
|
stopMonitoring();
|
|
@@ -1384,6 +1573,24 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1384
1573
|
stopMonitoring();
|
|
1385
1574
|
return;
|
|
1386
1575
|
}
|
|
1576
|
+
/* The catch-all for capture composition, on the one path every live capture stream's tick passes through - static captures included, which is why it precedes
|
|
1577
|
+
* the static-capture return below. Chrome composes the capture of a selected tab from the window's fitted presentation, and re-issuing the page's own declared
|
|
1578
|
+
* metrics moves the composition back to the emulated surface; firing it on a cadence means any disturbance, from any cause known or unknown, is corrected
|
|
1579
|
+
* within about a minute rather than lasting for the recording. The registry is read again here rather than reusing the native check below, which sits on the
|
|
1580
|
+
* far side of that return. The command uses currentPage, the reference tab replacement updates, and the stream context is re-established because an interval
|
|
1581
|
+
* callback does not inherit it - so a rejection logs under this stream's own prefix.
|
|
1582
|
+
*/
|
|
1583
|
+
reaffirmTickCounter++;
|
|
1584
|
+
if (((reaffirmTickCounter % SURFACE_REAFFIRM_TICK_INTERVAL) === 0) && (getStream(streamInfo.numericStreamId)?.identity.mode !== "native")) {
|
|
1585
|
+
void runWithStreamContext(streamContext, async () => {
|
|
1586
|
+
try {
|
|
1587
|
+
await reaffirmCaptureSurface(currentPage);
|
|
1588
|
+
}
|
|
1589
|
+
catch (error) {
|
|
1590
|
+
LOG.debug("browser:lifecycle", "Could not re-affirm the capture surface: %s.", formatError(error));
|
|
1591
|
+
}
|
|
1592
|
+
});
|
|
1593
|
+
}
|
|
1387
1594
|
// For static capture profiles (e.g., staticPage), there is no video element to monitor. Skip all video health checks and just emit a status update.
|
|
1388
1595
|
if (profile.staticCapture) {
|
|
1389
1596
|
emitStatusUpdate();
|
|
@@ -1392,15 +1599,15 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1392
1599
|
// For native streaming mode, monitor segment delivery health instead of video element state. We check the registry on each tick rather than caching the mode at
|
|
1393
1600
|
// startup because the streaming mode is set after the monitor starts (native streaming is attempted after setupStream returns).
|
|
1394
1601
|
const nativeEntry = getStream(streamInfo.numericStreamId);
|
|
1395
|
-
if (nativeEntry?.
|
|
1396
|
-
// Re-establish stream context for this interval tick before running the native health check. AsyncLocalStorage context is lost when entering
|
|
1397
|
-
//
|
|
1602
|
+
if (nativeEntry?.identity.mode === "native") {
|
|
1603
|
+
// Re-establish stream context for this interval tick before running the native health check. AsyncLocalStorage context is lost when entering the callback
|
|
1604
|
+
// timer, so without this wrapper the native path's non-debug warnings would emit without the stream-ID prefix. This mirrors the capture-mode branch below.
|
|
1398
1605
|
dispatchSerializedTick(() => runWithStreamContext(streamContext, async () => {
|
|
1399
1606
|
checkNativeStreamHealth(nativeEntry);
|
|
1400
1607
|
}));
|
|
1401
1608
|
return;
|
|
1402
1609
|
}
|
|
1403
|
-
// Re-establish stream context for this interval tick. AsyncLocalStorage context is lost when entering
|
|
1610
|
+
// Re-establish stream context for this interval tick. AsyncLocalStorage context is lost when entering the callback timer.
|
|
1404
1611
|
dispatchSerializedTick(() => runWithStreamContext(streamContext, async () => {
|
|
1405
1612
|
try {
|
|
1406
1613
|
// Early exit if the stream's abort signal has been triggered. This prevents wasted work when the stream is being terminated.
|
|
@@ -1410,7 +1617,7 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1410
1617
|
return;
|
|
1411
1618
|
}
|
|
1412
1619
|
// Capture current timestamp for all timing calculations in this check cycle.
|
|
1413
|
-
const now =
|
|
1620
|
+
const now = deps.clock.now();
|
|
1414
1621
|
// Gather current video state for analysis. The getVideoState helper encapsulates video element selection and returns all properties needed for health analysis.
|
|
1415
1622
|
// We catch frame detachment errors specifically to handle context invalidation differently from normal "video not found" cases.
|
|
1416
1623
|
let stateInfo = null;
|
|
@@ -1418,7 +1625,7 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1418
1625
|
try {
|
|
1419
1626
|
// A read issued while a timeout streak is open is a confirmation probe, so it carries the short bound. With no streak open the read takes the evaluate
|
|
1420
1627
|
// wrapper's default, which is the bound that detects a hung tab in the first place.
|
|
1421
|
-
stateInfo = await getVideoState(currentContext, selectorType, (consecutiveTimeouts > 0) ? UNRESPONSIVE_PROBE_TIMEOUT : undefined);
|
|
1628
|
+
stateInfo = await getVideoState(currentContext, selectorType, { clock: deps.clock, timeoutMs: (consecutiveTimeouts > 0) ? UNRESPONSIVE_PROBE_TIMEOUT : undefined });
|
|
1422
1629
|
}
|
|
1423
1630
|
catch (stateError) {
|
|
1424
1631
|
// Classify through the shared page-death predicate: the world the read ran in is gone, so the video may simply live in a context we no longer hold.
|
|
@@ -1437,7 +1644,7 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1437
1644
|
if (contextInvalidated) {
|
|
1438
1645
|
LOG.debug("recovery:context", "Re-searching for video after context invalidation.");
|
|
1439
1646
|
try {
|
|
1440
|
-
const newContext = await findVideoContext(currentPage, profile);
|
|
1647
|
+
const newContext = await findVideoContext(currentPage, profile, deps.clock);
|
|
1441
1648
|
const validationState = await validateVideoElement(newContext, selectorType);
|
|
1442
1649
|
if (validationState.found) {
|
|
1443
1650
|
LOG.info("Video found in new context after detachment. readyState=%s.", validationState.readyState);
|
|
@@ -1493,19 +1700,11 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1493
1700
|
// Check if we're within the buffering grace period (recently started buffering and haven't exceeded the threshold).
|
|
1494
1701
|
const withinBufferingGrace = isBuffering && bufferingStartTime && ((now - bufferingStartTime) < CONFIG.playback.bufferingGracePeriod);
|
|
1495
1702
|
// Check if we're within the recovery grace period (recently performed a recovery action and waiting for it to take effect).
|
|
1496
|
-
const withinRecoveryGrace =
|
|
1703
|
+
const withinRecoveryGrace = isWithinRecoveryGrace();
|
|
1497
1704
|
// Segment production monitoring: post-recovery verification and continuous size/staleness checks.
|
|
1498
1705
|
if (await monitorSegmentHealth(now, withinRecoveryGrace)) {
|
|
1499
1706
|
return;
|
|
1500
1707
|
}
|
|
1501
|
-
/* Re-minimize check. After recovery, the browser window may have been un-minimized by fullscreen actions. As soon as the stream is healthy (progressing without
|
|
1502
|
-
* issues), we re-minimize to reduce GPU usage.
|
|
1503
|
-
*/
|
|
1504
|
-
if (pendingReMinimize && isProgressing && !state.paused && !state.error && !state.ended) {
|
|
1505
|
-
LOG.debug("recovery:general", "Re-minimizing browser window after successful recovery.");
|
|
1506
|
-
pendingReMinimize = false;
|
|
1507
|
-
await minimizeWindow(currentPage);
|
|
1508
|
-
}
|
|
1509
1708
|
/* Fullscreen reinforcement. Some streaming sites (notably Hulu) revert the video to a mini-player or PiP layout in response to browser state changes such as
|
|
1510
1709
|
* window minimization or visibility events. Because the video continues playing normally in the smaller frame, no existing recovery condition is triggered - the
|
|
1511
1710
|
* health monitor sees healthy, progressing playback while the captured frame shows a small video in the corner of the viewport. We verify that the video fills
|
|
@@ -1582,9 +1781,10 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1582
1781
|
if (isProgressing && !state.paused && !state.ended && !state.error) {
|
|
1583
1782
|
// If a recovery was pending confirmation (L1/L2), log success now that we have healthy playback.
|
|
1584
1783
|
if ((metrics.currentRecoveryStartTime !== null) && (metrics.currentRecoveryMethod !== null)) {
|
|
1585
|
-
const
|
|
1784
|
+
const settledAt = deps.clock.now();
|
|
1785
|
+
const duration = formatRecoveryDuration(metrics.currentRecoveryStartTime, settledAt);
|
|
1586
1786
|
LOG.info("Recovered in %s via %s.", duration, metrics.currentRecoveryMethod);
|
|
1587
|
-
recordRecoverySuccess(metrics, metrics.currentRecoveryMethod);
|
|
1787
|
+
recordRecoverySuccess(metrics, metrics.currentRecoveryMethod, settledAt);
|
|
1588
1788
|
}
|
|
1589
1789
|
const healthyDuration = now - recoveryState.lastRecoveryTime;
|
|
1590
1790
|
if ((recoveryState.escalationLevel > 0) && (healthyDuration > CONFIG.playback.sustainedPlaybackRequired)) {
|
|
@@ -1628,11 +1828,24 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1628
1828
|
LOG.warn("Monitor check timed out (%s consecutive). Tab may be unresponsive.", consecutiveTimeouts);
|
|
1629
1829
|
// Update issue state so SSE clients can show the degraded state.
|
|
1630
1830
|
lastIssueType = "tab timing out";
|
|
1631
|
-
lastIssueTime =
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1831
|
+
lastIssueTime = deps.clock.now();
|
|
1832
|
+
/* After 3 consecutive timeouts the tab has stopped answering and only a replacement can save the stream. When one cannot be started, the stream is
|
|
1833
|
+
* unrecoverable: the ladder has nothing left to try, and leaving it in the registry would hold open the very relaunch that would cure a marked browser.
|
|
1834
|
+
* So the stream terminates through the breaker, exactly as the zombie case in handleExhaustedTabReplacement does.
|
|
1835
|
+
*
|
|
1836
|
+
* None of that runs inside the post-recovery grace window. A tab that has just been replaced is entitled to its settling time, and evaluate calls
|
|
1837
|
+
* against a page mid-establishment time out for reasons that are not a wedged tab - so escalating here would spend an attempt, or terminate a stream,
|
|
1838
|
+
* on evidence the window exists to discount. The timeout tally keeps climbing, so a genuinely wedged tab is acted on the moment the window closes.
|
|
1839
|
+
*/
|
|
1840
|
+
if ((consecutiveTimeouts >= 3) && !isWithinRecoveryGrace()) {
|
|
1841
|
+
if (canReplaceTab()) {
|
|
1842
|
+
LOG.warn("Tab unresponsive - recovering via %s.", RECOVERY_METHODS.tabReplacement);
|
|
1843
|
+
await executeTabReplacement("tab unresponsive");
|
|
1844
|
+
return;
|
|
1845
|
+
}
|
|
1846
|
+
LOG.error("Tab unresponsive and tab replacement is unavailable (%s) - terminating stream.", tabReplacementUnavailableReason());
|
|
1847
|
+
stopMonitoring();
|
|
1848
|
+
onCircuitBreak();
|
|
1636
1849
|
return;
|
|
1637
1850
|
}
|
|
1638
1851
|
// Emit status so SSE clients see current duration/memory even during timeout degradation (when consecutiveTimeouts < 3).
|
|
@@ -1657,7 +1870,7 @@ export function monitorPlaybackHealth(page, context, profile, url, streamId, str
|
|
|
1657
1870
|
// Log errors that escape the inner try/catch. In normal operation we should not reach here - if we do, there's a bug to investigate.
|
|
1658
1871
|
LOG.warn("Monitor tick error escaped inner try/catch: %s.", formatError(outerError));
|
|
1659
1872
|
}));
|
|
1660
|
-
}, CONFIG.playback.monitorInterval);
|
|
1873
|
+
}, CONFIG.playback.monitorInterval, { repeat: true });
|
|
1661
1874
|
/* The monitor's teardown: mark the interval cleared (so any in-flight async tick short-circuits) and clear it. Self-contained - it owns only the interval. Defined
|
|
1662
1875
|
* as a const so it is exposed as both dispose() and [Symbol.dispose].
|
|
1663
1876
|
*/
|