Safari audio stream

Safari iOS fix video stream
Safari video playback with sound
Some fixes

src/components/appMediaPlaybackController.ts

@@ -2,9 +2,15 @@ import { MTDocument } from "../types";
 import { $rootScope } from "../lib/utils";
 import appMessagesManager from "../lib/appManagers/appMessagesManager";
 import appDocsManager from "../lib/appManagers/appDocsManager";
+import { isSafari } from "../lib/config";
+import { CancellablePromise, deferredPromise } from "../lib/polyfill";
 
 // TODO: если удалить сообщение, и при этом аудио будет играть - оно не остановится, и можно будет по нему перейти вникуда
 
+// TODO: Safari: проверить стрим, включить его и сразу попробовать включить видео или другую песню
+// TODO: Safari: попробовать замаскировать подгрузку последнего чанка
+// TODO: Safari: пофиксить момент, когда заканчивается песня и пытаешься включить её заново - прогресс сразу в конце
+
 type HTMLMediaElement = HTMLAudioElement | HTMLVideoElement;
 
 type MediaType = 'voice' | 'audio' | 'round';
@@ -14,6 +20,8 @@ class AppMediaPlaybackController {
   private media: {[mid: string]: HTMLMediaElement} = {};
   private playingMedia: HTMLMediaElement;
 
+  private waitingMediaForLoad: {[mid: string]: CancellablePromise<void>} = {};
+  
   public willBePlayedMedia: HTMLMediaElement;
 
   private prevMid: number;
@@ -26,17 +34,22 @@ class AppMediaPlaybackController {
     document.body.append(this.container);
   }
 
-  public addMedia(doc: MTDocument, mid: number): HTMLMediaElement {
+  public addMedia(doc: MTDocument, mid: number, autoload = true): HTMLMediaElement {
     if(this.media[mid]) return this.media[mid];
 
     const media = document.createElement(doc.type == 'round' ? 'video' : 'audio');
     //const source = document.createElement('source');
     //source.type = doc.type == 'voice' && !opusDecodeController.isPlaySupported() ? 'audio/wav' : doc.mime_type;
 
-    media.autoplay = false;
+    media.dataset.mid = '' + mid;
+    media.dataset.type = doc.type;
+    
+    //media.autoplay = true;
     media.volume = 1;
     //media.append(source);
 
+    this.container.append(media);
+
     media.addEventListener('playing', () => {
       if(this.playingMedia != media) {
         if(this.playingMedia && !this.playingMedia.paused) {
@@ -68,20 +81,78 @@ class AppMediaPlaybackController {
 
     media.addEventListener('error', onError);
 
+    const deferred = deferredPromise<void>();
+    if(autoload) {
+      deferred.resolve();
+    } else {
+      this.waitingMediaForLoad[mid] = deferred;
+    }
+
+    // если что - загрузит voice или round заранее, так правильнее
     const downloadPromise: Promise<any> = !doc.supportsStreaming ? appDocsManager.downloadDocNew(doc.id) : Promise.resolve();
+    Promise.all([deferred, downloadPromise]).then(() => {
+      //media.autoplay = true;
+      //console.log('will set media url:', media, doc, doc.type, doc.url);
 
-    downloadPromise.then(() => {
+      if(doc.type == 'audio' && doc.supportsStreaming && isSafari) {
-      //if(doc.type != 'round') {
+        this.handleSafariStreamable(media);
-        this.container.append(media);
+      }
-      //}
 
-      //source.src = doc.url;
       media.src = doc.url;
     }, onError);
     
     return this.media[mid] = media;
   }
 
+  // safari подгрузит последний чанк и песня включится,
+  // при этом этот чанк нельзя руками отдать из SW, потому что браузер тогда теряется
+  private handleSafariStreamable(media: HTMLMediaElement) {
+    media.addEventListener('play', () => {
+      /* if(media.readyState == 4) { // https://developer.mozilla.org/ru/docs/Web/API/XMLHttpRequest/readyState
+        return;
+      } */
+
+      //media.volume = 0;
+      const currentTime = media.currentTime;
+      //this.setSafariBuffering(media, true);
+
+      media.addEventListener('progress', () => {
+        media.currentTime = media.duration - 1;
+
+        media.addEventListener('progress', () => {
+          media.currentTime = currentTime;
+          //media.volume = 1;
+          //this.setSafariBuffering(media, false);
+
+          if(!media.paused) {
+            media.play()/* .catch(() => {}) */;
+          }
+        }, {once: true});
+      }, {once: true});
+    }/* , {once: true} */);
+  }
+
+  public resolveWaitingForLoadMedia(mid: number) {
+    const promise = this.waitingMediaForLoad[mid];
+    if(promise) {
+      promise.resolve();
+      delete this.waitingMediaForLoad[mid];
+    }
+  }
+  
+  /**
+   * Only for audio
+   */
+  public isSafariBuffering(media: HTMLMediaElement) {
+    /// @ts-ignore
+    return !!media.safariBuffering;
+  }
+
+  private setSafariBuffering(media: HTMLMediaElement, value: boolean) {
+    // @ts-ignore
+    media.safariBuffering = value;
+  }
+
   onPause = (e: Event) => {
     $rootScope.$broadcast('audio_pause');
   };
@@ -89,8 +160,20 @@ class AppMediaPlaybackController {
   onEnded = (e: Event) => {
     this.onPause(e);
 
+    //console.log('on media end');
+
     if(this.nextMid) {
-      this.media[this.nextMid].play();
+      const media = this.media[this.nextMid];
+
+      /* if(isSafari) {
+        media.autoplay = true;
+      } */
+
+      this.resolveWaitingForLoadMedia(this.nextMid);
+
+      setTimeout(() => {
+        media.play()//.catch(() => {});
+      }, 0);
     }
   };
 
@@ -118,7 +201,7 @@ class AppMediaPlaybackController {
 
       [this.prevMid, this.nextMid].filter(Boolean).forEach(mid => {
         const message = appMessagesManager.getMessage(mid);
-        this.addMedia(message.media.document, mid);
+        this.addMedia(message.media.document, mid, false);
       });
       
       //console.log('loadSiblingsAudio', audio, type, mid, value, this.prevMid, this.nextMid);
@@ -143,10 +226,6 @@ class AppMediaPlaybackController {
   public willBePlayed(media: HTMLMediaElement) {
     this.willBePlayedMedia = media;
   }
-
-  public mediaExists(mid: number) {
-    return !!this.media[mid];
-  }
 }
 
 const appMediaPlaybackController = new AppMediaPlaybackController();

src/components/audio.ts

@@ -5,8 +5,9 @@ import ProgressivePreloader from "./preloader";
 import { MediaProgressLine } from "../lib/mediaPlayer";
 import appMediaPlaybackController from "./appMediaPlaybackController";
 import { MTDocument } from "../types";
-import { mediaSizes } from "../lib/config";
+import { mediaSizes, isSafari } from "../lib/config";
 import { Download } from "../lib/appManagers/appDownloadManager";
+import { deferredPromise, CancellablePromise } from "../lib/polyfill";
 
 // https://github.com/LonamiWebs/Telethon/blob/4393ec0b83d511b6a20d8a20334138730f084375/telethon/utils.py#L1285
 export function decodeWaveform(waveform: Uint8Array | number[]) {
@@ -312,8 +313,8 @@ export default class AudioElement extends HTMLElement {
     const audioTimeDiv = this.querySelector('.audio-time') as HTMLDivElement;
     audioTimeDiv.innerHTML = durationStr;
 
-    const onLoad = () => {
+    const onLoad = (autoload = true) => {
-      const audio = this.audio = appMediaPlaybackController.addMedia(doc, mid);
+      const audio = this.audio = appMediaPlaybackController.addMedia(doc, mid, autoload);
 
       this.onTypeDisconnect = onTypeLoad();
       
@@ -333,7 +334,7 @@ export default class AudioElement extends HTMLElement {
       }
 
       toggle.addEventListener('click', () => {
-        if(audio.paused) audio.play();
+        if(audio.paused) audio.play().catch(() => {});
         else audio.pause();
       });
       
@@ -343,6 +344,7 @@ export default class AudioElement extends HTMLElement {
       });
 
       this.addAudioListener('timeupdate', () => {
+        if(appMediaPlaybackController.isSafariBuffering(audio)) return;
         audioTimeDiv.innerText = String(audio.currentTime | 0).toHHMMSS(true) + ' / ' + durationStr;
       });
 
@@ -390,11 +392,14 @@ export default class AudioElement extends HTMLElement {
         this.addEventListener('click', onClick);
         this.click();
       } else {
-        if(appMediaPlaybackController.mediaExists(mid)) { // чтобы показать прогресс, если аудио уже было скачано
+        onLoad(false);
-          onLoad();
+
-        } else {
+        //if(appMediaPlaybackController.mediaExists(mid)) { // чтобы показать прогресс, если аудио уже было скачано
+          //onLoad();
+        //} else {
           const r = () => {
-            onLoad();
+            //onLoad();
+            appMediaPlaybackController.resolveWaitingForLoadMedia(mid);
   
             appMediaPlaybackController.willBePlayed(this.audio); // prepare for loading audio
   
@@ -402,6 +407,11 @@ export default class AudioElement extends HTMLElement {
               preloader = new ProgressivePreloader(null, false);
             }
 
+            if(isSafari) {
+              this.audio.autoplay = true;
+              this.audio.play().catch(() => {});
+            }
+    
             preloader.attach(downloadDiv);
             this.append(downloadDiv);
     
@@ -422,7 +432,7 @@ export default class AudioElement extends HTMLElement {
           };
   
           this.addEventListener('click', r, {once: true});
-        }
+        //}
       }
     } else {
       this.preloader.attach(downloadDiv, false);

src/components/wrappers.ts

@@ -62,8 +62,10 @@ export function wrapVideo({doc, container, message, boxWidth, boxHeight, withTai
   } */
 
   const video = document.createElement('video');
-  if(doc.type == 'round') {
   video.muted = true;
+  video.setAttribute('playsinline', '');
+  if(doc.type == 'round') {
+    //video.muted = true;
     const globalVideo = appMediaPlaybackController.addMedia(doc, message.mid);
 
     video.addEventListener('canplay', () => {
@@ -121,6 +123,8 @@ export function wrapVideo({doc, container, message, boxWidth, boxHeight, withTai
     globalVideo.addEventListener('pause', onGlobalPause);
     video.addEventListener('play', onVideoPlay);
     video.addEventListener('pause', onVideoPause);
+  } else {
+    video.autoplay = true; // для safari
   }
   
   let img: HTMLImageElement;
@@ -224,8 +228,6 @@ export function wrapVideo({doc, container, message, boxWidth, boxHeight, withTai
       renderImageFromUrl(video, doc.url);
     //}
 
-    video.setAttribute('playsinline', '');
-
     /* if(!container.parentElement) {
       container.append(video);
     } */

src/lib/appManagers/appDocsManager.ts

@@ -130,6 +130,9 @@ class AppDocsManager {
       }
     }
 
+    // for testing purposes
+    //doc.supportsStreaming = false;
+    
     if(!doc.file_name) {
       doc.file_name = '';
     }

src/lib/appManagers/appMediaViewer.ts

@@ -923,16 +923,26 @@ export class AppMediaViewer {
     if(isVideo) {
       ////////this.log('will wrap video', media, size);
 
+      // потому что для safari нужно создать элемент из event'а
+      const video = document.createElement('video');
+
       setMoverPromise = this.setMoverToTarget(target, false, fromRight).then(({onAnimationEnd}) => {
       //return; // set and don't move
       //if(wasActive) return;
         //return;
 
         const div = mover.firstElementChild && mover.firstElementChild.classList.contains('media-viewer-aspecter') ? mover.firstElementChild : mover;
-        const video = mover.querySelector('video') || document.createElement('video');
+        //const video = mover.querySelector('video') || document.createElement('video');
+
+        const moverVideo = mover.querySelector('video');
+        if(moverVideo) {
+          moverVideo.remove();
+        }
+
         //video.src = '';
 
         video.setAttribute('playsinline', '');
+        video.autoplay = true;
         if(media.type == 'gif') {
           video.muted = true;
           video.autoplay = true;

src/lib/appManagers/appSidebarRight.ts

@@ -223,7 +223,7 @@ export class AppSidebarRight extends SidebarSlider {
       let ids = Object.keys(this.mediaDivsByIDs).map(k => +k).sort((a, b) => a - b);
       let idx = ids.findIndex(i => i == messageID);
       
-      let targets = ids.map(id => ({element: this.mediaDivsByIDs[id].firstElementChild as HTMLElement, mid: id}));
+      let targets = ids.map(id => ({element: this.mediaDivsByIDs[id]/* .firstElementChild */ as HTMLElement, mid: id}));
       
       appMediaViewer.openMedia(message, target, false, this.sidebarEl, targets.slice(idx + 1).reverse(), targets.slice(0, idx).reverse(), true);
     });

src/lib/mediaPlayer.ts

@@ -1,5 +1,6 @@
 import { cancelEvent } from "./utils";
 import { touchSupport } from "./config";
+import appMediaPlaybackController from "../components/appMediaPlaybackController";
 
 export class ProgressLine {
   public container: HTMLDivElement;
@@ -183,6 +184,7 @@ export class MediaProgressLine extends ProgressLine {
   }
 
   protected setLoadProgress() {
+    if(appMediaPlaybackController.isSafariBuffering(this.media)) return;
     const buf = this.media.buffered;
     const numRanges = buf.length;
 
@@ -214,6 +216,7 @@ export class MediaProgressLine extends ProgressLine {
   }
 
   public setProgress() {
+    if(appMediaPlaybackController.isSafariBuffering(this.media)) return;
     const currentTime = this.media.currentTime;
 
     super.setProgress(currentTime);
@@ -270,7 +273,7 @@ export default class VideoPlayer {
       controls.prepend(this.progress.container);
     }
 
-    if(play && video.paused) {
+    if(play/*  && video.paused */) {
       const promise = video.play();
       promise.catch((err: Error) => {
         if(err.name == 'NotAllowedError') {

src/lib/mtproto/mtproto.service.ts

@@ -5,7 +5,7 @@ import type { InputFileLocation, FileLocation, UploadFile, WorkerTaskTemplate }
 import { deferredPromise, CancellablePromise } from '../polyfill';
 import { notifySomeone } from '../../helpers/context';
 
-const log = logger('SW', LogLevels.error);
+const log = logger('SW', LogLevels.error/*  | LogLevels.debug | LogLevels.log */);
 const ctx = self as any as ServiceWorkerGlobalScope;
 
 const deferredPromises: {[taskID: number]: CancellablePromise<any>} = {};
@@ -44,11 +44,19 @@ const onFetch = (event: FetchEvent): void => {
     switch(scope) {
       case 'stream': {
         const range = parseRange(event.request.headers.get('Range'));
-        const [offset, end] = range;
+        let [offset, end] = range;
   
         const info: DownloadOptions = JSON.parse(decodeURIComponent(params));
         //const fileName = getFileNameByLocation(info.location);
 
+        const limitPart = STREAM_CHUNK_UPPER_LIMIT;
+
+        /* if(info.size > limitPart && isSafari && offset == limitPart) {
+          //end = info.size - 1;
+          //offset = info.size - 1 - limitPart;
+          offset = info.size - (info.size % limitPart);
+        } */
+  
         log.debug('[stream]', url, offset, end);
   
         event.respondWith(Promise.race([
@@ -61,10 +69,10 @@ const onFetch = (event: FetchEvent): void => {
               return resolve(possibleResponse);
             }
   
-            const limit = end && end < STREAM_CHUNK_UPPER_LIMIT ? alignLimit(end - offset + 1) : STREAM_CHUNK_UPPER_LIMIT;
+            const limit = end && end < limitPart ? alignLimit(end - offset + 1) : limitPart;
             const alignedOffset = alignOffset(offset, limit);
   
-            log.debug('[stream] requestFilePart:', info.dcID, info.location, alignedOffset, limit);
+            log.debug('[stream] requestFilePart:', /* info.dcID, info.location, */ alignedOffset, limit);
 
             const task: ServiceWorkerTask = {
               type: 'requestFilePart',
@@ -77,7 +85,7 @@ const onFetch = (event: FetchEvent): void => {
             deferred.then(result => {
               let ab = result.bytes;
               
-              //log.debug('[stream] requestFilePart result:', result);
+              log.debug('[stream] requestFilePart result:', result);
   
               const headers: Record<string, string> = {
                 'Accept-Ranges': 'bytes',

webpack.common.js

@@ -6,7 +6,7 @@ const postcssPresetEnv = require('postcss-preset-env');
 const ServiceWorkerWebpackPlugin = require('serviceworker-webpack-plugin');
 const fs = require('fs');
 
-const allowedIPs = ['195.66.140.39', '192.168.31.144', '127.0.0.1', '192.168.31.1', '192.168.31.192', '176.100.18.181', '46.219.250.22', '193.42.119.184'];
+const allowedIPs = ['195.66.140.39', '192.168.31.144', '127.0.0.1', '192.168.31.1', '192.168.31.192', '176.100.18.181', '46.219.250.22', '193.42.119.184', '46.133.168.67'];
 const devMode = process.env.NODE_ENV !== 'production';
 const useLocal = false;
 
@@ -111,7 +111,8 @@ module.exports = {
           IP = req.connection.remoteAddress.split(':').pop();
         }
 
-        if(!allowedIPs.includes(IP) && !/^192\.168\.\d{1,3}\.\d{1,3}$/.test(IP)) {
+        // last is ODESSA
+        if(!allowedIPs.includes(IP) && !/^192\.168\.\d{1,3}\.\d{1,3}$/.test(IP) && !/^88\.155\.57\.\d{1,3}$/.test(IP)) {
           console.log('Bad IP connecting: ' + IP, req.url);
           res.status(404).send('Nothing interesting here.');
         } else {