From 9ae707aa7b486c420d1d8bf60d0873e83dc46a76 Mon Sep 17 00:00:00 2001 From: morethanwords Date: Sun, 30 Aug 2020 13:43:57 +0300 Subject: [PATCH] Safari audio stream Safari iOS fix video stream Safari video playback with sound Some fixes --- src/components/appMediaPlaybackController.ts | 109 ++++++++++++++++--- src/components/audio.ts | 28 +++-- src/components/wrappers.ts | 10 +- src/lib/appManagers/appDocsManager.ts | 5 +- src/lib/appManagers/appMediaViewer.ts | 12 +- src/lib/appManagers/appSidebarRight.ts | 2 +- src/lib/mediaPlayer.ts | 5 +- src/lib/mtproto/mtproto.service.ts | 18 ++- webpack.common.js | 7 +- 9 files changed, 156 insertions(+), 40 deletions(-) diff --git a/src/components/appMediaPlaybackController.ts b/src/components/appMediaPlaybackController.ts index e7c00d13..faa4c055 100644 --- a/src/components/appMediaPlaybackController.ts +++ b/src/components/appMediaPlaybackController.ts @@ -2,9 +2,15 @@ import { MTDocument } from "../types"; import { $rootScope } from "../lib/utils"; import appMessagesManager from "../lib/appManagers/appMessagesManager"; import appDocsManager from "../lib/appManagers/appDocsManager"; +import { isSafari } from "../lib/config"; +import { CancellablePromise, deferredPromise } from "../lib/polyfill"; // TODO: если удалить сообщение, и при этом аудио будет играть - оно не остановится, и можно будет по нему перейти вникуда +// TODO: Safari: проверить стрим, включить его и сразу попробовать включить видео или другую песню +// TODO: Safari: попробовать замаскировать подгрузку последнего чанка +// TODO: Safari: пофиксить момент, когда заканчивается песня и пытаешься включить её заново - прогресс сразу в конце + type HTMLMediaElement = HTMLAudioElement | HTMLVideoElement; type MediaType = 'voice' | 'audio' | 'round'; @@ -13,6 +19,8 @@ class AppMediaPlaybackController { private container: HTMLElement; private media: {[mid: string]: HTMLMediaElement} = {}; private playingMedia: HTMLMediaElement; + + private waitingMediaForLoad: {[mid: string]: CancellablePromise} = {}; public willBePlayedMedia: HTMLMediaElement; @@ -26,17 +34,22 @@ class AppMediaPlaybackController { document.body.append(this.container); } - public addMedia(doc: MTDocument, mid: number): HTMLMediaElement { + public addMedia(doc: MTDocument, mid: number, autoload = true): HTMLMediaElement { if(this.media[mid]) return this.media[mid]; const media = document.createElement(doc.type == 'round' ? 'video' : 'audio'); //const source = document.createElement('source'); //source.type = doc.type == 'voice' && !opusDecodeController.isPlaySupported() ? 'audio/wav' : doc.mime_type; + + media.dataset.mid = '' + mid; + media.dataset.type = doc.type; - media.autoplay = false; + //media.autoplay = true; media.volume = 1; //media.append(source); + this.container.append(media); + media.addEventListener('playing', () => { if(this.playingMedia != media) { if(this.playingMedia && !this.playingMedia.paused) { @@ -68,20 +81,78 @@ class AppMediaPlaybackController { media.addEventListener('error', onError); + const deferred = deferredPromise(); + if(autoload) { + deferred.resolve(); + } else { + this.waitingMediaForLoad[mid] = deferred; + } + + // если что - загрузит voice или round заранее, так правильнее const downloadPromise: Promise = !doc.supportsStreaming ? appDocsManager.downloadDocNew(doc.id) : Promise.resolve(); + Promise.all([deferred, downloadPromise]).then(() => { + //media.autoplay = true; + //console.log('will set media url:', media, doc, doc.type, doc.url); - downloadPromise.then(() => { - //if(doc.type != 'round') { - this.container.append(media); - //} + if(doc.type == 'audio' && doc.supportsStreaming && isSafari) { + this.handleSafariStreamable(media); + } - //source.src = doc.url; media.src = doc.url; }, onError); - + return this.media[mid] = media; } + // safari подгрузит последний чанк и песня включится, + // при этом этот чанк нельзя руками отдать из SW, потому что браузер тогда теряется + private handleSafariStreamable(media: HTMLMediaElement) { + media.addEventListener('play', () => { + /* if(media.readyState == 4) { // https://developer.mozilla.org/ru/docs/Web/API/XMLHttpRequest/readyState + return; + } */ + + //media.volume = 0; + const currentTime = media.currentTime; + //this.setSafariBuffering(media, true); + + media.addEventListener('progress', () => { + media.currentTime = media.duration - 1; + + media.addEventListener('progress', () => { + media.currentTime = currentTime; + //media.volume = 1; + //this.setSafariBuffering(media, false); + + if(!media.paused) { + media.play()/* .catch(() => {}) */; + } + }, {once: true}); + }, {once: true}); + }/* , {once: true} */); + } + + public resolveWaitingForLoadMedia(mid: number) { + const promise = this.waitingMediaForLoad[mid]; + if(promise) { + promise.resolve(); + delete this.waitingMediaForLoad[mid]; + } + } + + /** + * Only for audio + */ + public isSafariBuffering(media: HTMLMediaElement) { + /// @ts-ignore + return !!media.safariBuffering; + } + + private setSafariBuffering(media: HTMLMediaElement, value: boolean) { + // @ts-ignore + media.safariBuffering = value; + } + onPause = (e: Event) => { $rootScope.$broadcast('audio_pause'); }; @@ -89,8 +160,20 @@ class AppMediaPlaybackController { onEnded = (e: Event) => { this.onPause(e); + //console.log('on media end'); + if(this.nextMid) { - this.media[this.nextMid].play(); + const media = this.media[this.nextMid]; + + /* if(isSafari) { + media.autoplay = true; + } */ + + this.resolveWaitingForLoadMedia(this.nextMid); + + setTimeout(() => { + media.play()//.catch(() => {}); + }, 0); } }; @@ -106,7 +189,7 @@ class AppMediaPlaybackController { if(this.playingMedia != media) { return; } - + for(let m of value.history) { if(m > mid) { this.nextMid = m; @@ -118,7 +201,7 @@ class AppMediaPlaybackController { [this.prevMid, this.nextMid].filter(Boolean).forEach(mid => { const message = appMessagesManager.getMessage(mid); - this.addMedia(message.media.document, mid); + this.addMedia(message.media.document, mid, false); }); //console.log('loadSiblingsAudio', audio, type, mid, value, this.prevMid, this.nextMid); @@ -143,10 +226,6 @@ class AppMediaPlaybackController { public willBePlayed(media: HTMLMediaElement) { this.willBePlayedMedia = media; } - - public mediaExists(mid: number) { - return !!this.media[mid]; - } } const appMediaPlaybackController = new AppMediaPlaybackController(); diff --git a/src/components/audio.ts b/src/components/audio.ts index fb567ca4..522c40f3 100644 --- a/src/components/audio.ts +++ b/src/components/audio.ts @@ -5,8 +5,9 @@ import ProgressivePreloader from "./preloader"; import { MediaProgressLine } from "../lib/mediaPlayer"; import appMediaPlaybackController from "./appMediaPlaybackController"; import { MTDocument } from "../types"; -import { mediaSizes } from "../lib/config"; +import { mediaSizes, isSafari } from "../lib/config"; import { Download } from "../lib/appManagers/appDownloadManager"; +import { deferredPromise, CancellablePromise } from "../lib/polyfill"; // https://github.com/LonamiWebs/Telethon/blob/4393ec0b83d511b6a20d8a20334138730f084375/telethon/utils.py#L1285 export function decodeWaveform(waveform: Uint8Array | number[]) { @@ -312,8 +313,8 @@ export default class AudioElement extends HTMLElement { const audioTimeDiv = this.querySelector('.audio-time') as HTMLDivElement; audioTimeDiv.innerHTML = durationStr; - const onLoad = () => { - const audio = this.audio = appMediaPlaybackController.addMedia(doc, mid); + const onLoad = (autoload = true) => { + const audio = this.audio = appMediaPlaybackController.addMedia(doc, mid, autoload); this.onTypeDisconnect = onTypeLoad(); @@ -333,7 +334,7 @@ export default class AudioElement extends HTMLElement { } toggle.addEventListener('click', () => { - if(audio.paused) audio.play(); + if(audio.paused) audio.play().catch(() => {}); else audio.pause(); }); @@ -343,6 +344,7 @@ export default class AudioElement extends HTMLElement { }); this.addAudioListener('timeupdate', () => { + if(appMediaPlaybackController.isSafariBuffering(audio)) return; audioTimeDiv.innerText = String(audio.currentTime | 0).toHHMMSS(true) + ' / ' + durationStr; }); @@ -390,17 +392,25 @@ export default class AudioElement extends HTMLElement { this.addEventListener('click', onClick); this.click(); } else { - if(appMediaPlaybackController.mediaExists(mid)) { // чтобы показать прогресс, если аудио уже было скачано - onLoad(); - } else { + onLoad(false); + + //if(appMediaPlaybackController.mediaExists(mid)) { // чтобы показать прогресс, если аудио уже было скачано + //onLoad(); + //} else { const r = () => { - onLoad(); + //onLoad(); + appMediaPlaybackController.resolveWaitingForLoadMedia(mid); appMediaPlaybackController.willBePlayed(this.audio); // prepare for loading audio if(!preloader) { preloader = new ProgressivePreloader(null, false); } + + if(isSafari) { + this.audio.autoplay = true; + this.audio.play().catch(() => {}); + } preloader.attach(downloadDiv); this.append(downloadDiv); @@ -422,7 +432,7 @@ export default class AudioElement extends HTMLElement { }; this.addEventListener('click', r, {once: true}); - } + //} } } else { this.preloader.attach(downloadDiv, false); diff --git a/src/components/wrappers.ts b/src/components/wrappers.ts index cba46718..fbc71fe5 100644 --- a/src/components/wrappers.ts +++ b/src/components/wrappers.ts @@ -62,8 +62,10 @@ export function wrapVideo({doc, container, message, boxWidth, boxHeight, withTai } */ const video = document.createElement('video'); + video.muted = true; + video.setAttribute('playsinline', ''); if(doc.type == 'round') { - video.muted = true; + //video.muted = true; const globalVideo = appMediaPlaybackController.addMedia(doc, message.mid); video.addEventListener('canplay', () => { @@ -121,6 +123,8 @@ export function wrapVideo({doc, container, message, boxWidth, boxHeight, withTai globalVideo.addEventListener('pause', onGlobalPause); video.addEventListener('play', onVideoPlay); video.addEventListener('pause', onVideoPause); + } else { + video.autoplay = true; // для safari } let img: HTMLImageElement; @@ -224,12 +228,10 @@ export function wrapVideo({doc, container, message, boxWidth, boxHeight, withTai renderImageFromUrl(video, doc.url); //} - video.setAttribute('playsinline', ''); - /* if(!container.parentElement) { container.append(video); } */ - + if(doc.type == 'gif'/* || true */) { video.muted = true; video.loop = true; diff --git a/src/lib/appManagers/appDocsManager.ts b/src/lib/appManagers/appDocsManager.ts index 2f409f60..a97ac0fd 100644 --- a/src/lib/appManagers/appDocsManager.ts +++ b/src/lib/appManagers/appDocsManager.ts @@ -124,11 +124,14 @@ class AppDocsManager { if((doc.type == 'gif' && doc.size > 8e6) || doc.type == 'audio' || doc.type == 'video') { doc.supportsStreaming = true; - + if(!doc.url) { doc.url = this.getFileURL(doc); } } + + // for testing purposes + //doc.supportsStreaming = false; if(!doc.file_name) { doc.file_name = ''; diff --git a/src/lib/appManagers/appMediaViewer.ts b/src/lib/appManagers/appMediaViewer.ts index d6dc8cb2..17ef8e7f 100644 --- a/src/lib/appManagers/appMediaViewer.ts +++ b/src/lib/appManagers/appMediaViewer.ts @@ -923,16 +923,26 @@ export class AppMediaViewer { if(isVideo) { ////////this.log('will wrap video', media, size); + // потому что для safari нужно создать элемент из event'а + const video = document.createElement('video'); + setMoverPromise = this.setMoverToTarget(target, false, fromRight).then(({onAnimationEnd}) => { //return; // set and don't move //if(wasActive) return; //return; const div = mover.firstElementChild && mover.firstElementChild.classList.contains('media-viewer-aspecter') ? mover.firstElementChild : mover; - const video = mover.querySelector('video') || document.createElement('video'); + //const video = mover.querySelector('video') || document.createElement('video'); + + const moverVideo = mover.querySelector('video'); + if(moverVideo) { + moverVideo.remove(); + } + //video.src = ''; video.setAttribute('playsinline', ''); + video.autoplay = true; if(media.type == 'gif') { video.muted = true; video.autoplay = true; diff --git a/src/lib/appManagers/appSidebarRight.ts b/src/lib/appManagers/appSidebarRight.ts index 1e207cf6..3ea6316a 100644 --- a/src/lib/appManagers/appSidebarRight.ts +++ b/src/lib/appManagers/appSidebarRight.ts @@ -223,7 +223,7 @@ export class AppSidebarRight extends SidebarSlider { let ids = Object.keys(this.mediaDivsByIDs).map(k => +k).sort((a, b) => a - b); let idx = ids.findIndex(i => i == messageID); - let targets = ids.map(id => ({element: this.mediaDivsByIDs[id].firstElementChild as HTMLElement, mid: id})); + let targets = ids.map(id => ({element: this.mediaDivsByIDs[id]/* .firstElementChild */ as HTMLElement, mid: id})); appMediaViewer.openMedia(message, target, false, this.sidebarEl, targets.slice(idx + 1).reverse(), targets.slice(0, idx).reverse(), true); }); diff --git a/src/lib/mediaPlayer.ts b/src/lib/mediaPlayer.ts index e7b0e6d2..d6294f2d 100644 --- a/src/lib/mediaPlayer.ts +++ b/src/lib/mediaPlayer.ts @@ -1,5 +1,6 @@ import { cancelEvent } from "./utils"; import { touchSupport } from "./config"; +import appMediaPlaybackController from "../components/appMediaPlaybackController"; export class ProgressLine { public container: HTMLDivElement; @@ -183,6 +184,7 @@ export class MediaProgressLine extends ProgressLine { } protected setLoadProgress() { + if(appMediaPlaybackController.isSafariBuffering(this.media)) return; const buf = this.media.buffered; const numRanges = buf.length; @@ -214,6 +216,7 @@ export class MediaProgressLine extends ProgressLine { } public setProgress() { + if(appMediaPlaybackController.isSafariBuffering(this.media)) return; const currentTime = this.media.currentTime; super.setProgress(currentTime); @@ -270,7 +273,7 @@ export default class VideoPlayer { controls.prepend(this.progress.container); } - if(play && video.paused) { + if(play/* && video.paused */) { const promise = video.play(); promise.catch((err: Error) => { if(err.name == 'NotAllowedError') { diff --git a/src/lib/mtproto/mtproto.service.ts b/src/lib/mtproto/mtproto.service.ts index bd4f8e27..fa4bb0ab 100644 --- a/src/lib/mtproto/mtproto.service.ts +++ b/src/lib/mtproto/mtproto.service.ts @@ -5,7 +5,7 @@ import type { InputFileLocation, FileLocation, UploadFile, WorkerTaskTemplate } import { deferredPromise, CancellablePromise } from '../polyfill'; import { notifySomeone } from '../../helpers/context'; -const log = logger('SW', LogLevels.error); +const log = logger('SW', LogLevels.error/* | LogLevels.debug | LogLevels.log */); const ctx = self as any as ServiceWorkerGlobalScope; const deferredPromises: {[taskID: number]: CancellablePromise} = {}; @@ -44,10 +44,18 @@ const onFetch = (event: FetchEvent): void => { switch(scope) { case 'stream': { const range = parseRange(event.request.headers.get('Range')); - const [offset, end] = range; + let [offset, end] = range; const info: DownloadOptions = JSON.parse(decodeURIComponent(params)); //const fileName = getFileNameByLocation(info.location); + + const limitPart = STREAM_CHUNK_UPPER_LIMIT; + + /* if(info.size > limitPart && isSafari && offset == limitPart) { + //end = info.size - 1; + //offset = info.size - 1 - limitPart; + offset = info.size - (info.size % limitPart); + } */ log.debug('[stream]', url, offset, end); @@ -61,10 +69,10 @@ const onFetch = (event: FetchEvent): void => { return resolve(possibleResponse); } - const limit = end && end < STREAM_CHUNK_UPPER_LIMIT ? alignLimit(end - offset + 1) : STREAM_CHUNK_UPPER_LIMIT; + const limit = end && end < limitPart ? alignLimit(end - offset + 1) : limitPart; const alignedOffset = alignOffset(offset, limit); - log.debug('[stream] requestFilePart:', info.dcID, info.location, alignedOffset, limit); + log.debug('[stream] requestFilePart:', /* info.dcID, info.location, */ alignedOffset, limit); const task: ServiceWorkerTask = { type: 'requestFilePart', @@ -77,7 +85,7 @@ const onFetch = (event: FetchEvent): void => { deferred.then(result => { let ab = result.bytes; - //log.debug('[stream] requestFilePart result:', result); + log.debug('[stream] requestFilePart result:', result); const headers: Record = { 'Accept-Ranges': 'bytes', diff --git a/webpack.common.js b/webpack.common.js index 021d572a..35548477 100644 --- a/webpack.common.js +++ b/webpack.common.js @@ -6,7 +6,7 @@ const postcssPresetEnv = require('postcss-preset-env'); const ServiceWorkerWebpackPlugin = require('serviceworker-webpack-plugin'); const fs = require('fs'); -const allowedIPs = ['195.66.140.39', '192.168.31.144', '127.0.0.1', '192.168.31.1', '192.168.31.192', '176.100.18.181', '46.219.250.22', '193.42.119.184']; +const allowedIPs = ['195.66.140.39', '192.168.31.144', '127.0.0.1', '192.168.31.1', '192.168.31.192', '176.100.18.181', '46.219.250.22', '193.42.119.184', '46.133.168.67']; const devMode = process.env.NODE_ENV !== 'production'; const useLocal = false; @@ -110,8 +110,9 @@ module.exports = { } else { IP = req.connection.remoteAddress.split(':').pop(); } - - if(!allowedIPs.includes(IP) && !/^192\.168\.\d{1,3}\.\d{1,3}$/.test(IP)) { + + // last is ODESSA + if(!allowedIPs.includes(IP) && !/^192\.168\.\d{1,3}\.\d{1,3}$/.test(IP) && !/^88\.155\.57\.\d{1,3}$/.test(IP)) { console.log('Bad IP connecting: ' + IP, req.url); res.status(404).send('Nothing interesting here.'); } else {