diff --git a/.eslintrc.js b/.eslintrc.js index 09bac06f..765d14ac 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -6,9 +6,9 @@ module.exports = { extends: [], parser: '@typescript-eslint/parser', parserOptions: { - // 'ecmaVersion': 'latest', - // 'sourceType': 'module' - project: ['./tsconfig.json'] + 'ecmaVersion': 'latest', + 'sourceType': 'module' + // project: ['./tsconfig.json'] }, plugins: [ '@typescript-eslint' diff --git a/src/components/chat/bubbles.ts b/src/components/chat/bubbles.ts index ec595bdf..1cbcf858 100644 --- a/src/components/chat/bubbles.ts +++ b/src/components/chat/bubbles.ts @@ -522,6 +522,7 @@ export default class ChatBubbles { (element as any).onLoad(true); } else { element.dataset.docId = '' + doc.id; + (element as any).doc = doc; } } } else if(poll) { diff --git a/src/components/wrappers/document.ts b/src/components/wrappers/document.ts index 3ef9e59e..3b5e60b7 100644 --- a/src/components/wrappers/document.ts +++ b/src/components/wrappers/document.ts @@ -90,6 +90,7 @@ export default async function wrapDocument({message, withTime, fontWeight, voice const docDiv = document.createElement('div'); docDiv.classList.add('document', `ext-${ext}`); docDiv.dataset.docId = '' + doc.id; + (docDiv as any).doc = doc; // return docDiv; @@ -236,25 +237,28 @@ export default async function wrapDocument({message, withTime, fontWeight, voice // b && b.classList.add('hide'); - let d = formatBytes(0); + const format = (bytes: number) => formatBytes(bytes); + let d = format(0); bytesContainer.style.visibility = 'hidden'; // bytesContainer.replaceWith(sizeContainer); sizeContainer.append(d, bytesJoiner, _bytesContainer); bytesContainer.parentElement.append(sizeContainer); promise.addNotifyListener((progress: Progress) => { - const _d = formatBytes(progress.done); + const _d = format(progress.done); d.replaceWith(_d); d = _d; }); }; - const load = async(e?: Event) => { + // ! DO NOT USE ASYNC/AWAIT HERE ! SAFARI WON'T LET DOWNLOAD THE FILE BECAUSE OF ASYNC + const load = (e?: Event) => { const save = !e || e.isTrusted; - const doc = await managers.appDocsManager.getDoc(docDiv.dataset.docId); + const doc = (docDiv as any).doc; + // const doc = await managers.appDocsManager.getDoc(docDiv.dataset.docId); let download: CancellablePromise; const queueId = appImManager.chat.bubbles ? appImManager.chat.bubbles.lazyLoadQueue.queueId : undefined; if(!save) { - download = appDownloadManager.downloadMediaVoid({media: doc, queueId}); + download = appDownloadManager.downloadToDisc({media: doc, queueId}, true); } else if(doc.type === 'pdf') { const canOpenAfter = /* managers.appDocsManager.downloading.has(doc.id) || */!preloader || preloader.detached; download = appDownloadManager.downloadMediaURL({media: doc, queueId}); @@ -282,10 +286,10 @@ export default async function wrapDocument({message, withTime, fontWeight, voice } }; - const {fileName: downloadFileName} = getDownloadMediaDetails({media: doc}); + const {fileName: downloadFileName} = getDownloadMediaDetails({media: doc, downloadId: '1'}); if(await managers.apiFileManager.isDownloading(downloadFileName)) { downloadDiv = docDiv.querySelector('.document-download') || icoDiv; - const promise = appDownloadManager.downloadMediaVoid({media: doc}); + const promise = appDownloadManager.downloadToDisc({media: doc}, true); preloader = new ProgressivePreloader(); preloader.attach(downloadDiv, false, promise); diff --git a/src/global.d.ts b/src/global.d.ts index e2744251..271540c0 100644 --- a/src/global.d.ts +++ b/src/global.d.ts @@ -30,7 +30,7 @@ declare global { type FiltersError = 'PINNED_DIALOGS_TOO_MUCH'; type LocalFileError = ApiFileManagerError | ReferenceError | StorageError; - type LocalErrorType = LocalFileError | NetworkerError | FiltersError | 'UNKNOWN'; + type LocalErrorType = LocalFileError | NetworkerError | FiltersError | 'UNKNOWN' | 'NO_DOC'; type ServerErrorType = 'FILE_REFERENCE_EXPIRED' | 'SESSION_REVOKED' | 'AUTH_KEY_DUPLICATED' | 'SESSION_PASSWORD_NEEDED' | 'CONNECTION_NOT_INITED' | 'ERROR_EMPTY' | 'MTPROTO_CLUSTER_INVALID' | diff --git a/src/helpers/fileName.ts b/src/helpers/fileName.ts index ef7d0b78..569d9064 100644 --- a/src/helpers/fileName.ts +++ b/src/helpers/fileName.ts @@ -58,7 +58,7 @@ export function getFileNameByLocation(location: InputFileLocation | InputWebFile } } - return str + (options.downloadId || '') + (ext ? '.' + ext : ext); + return str + (options.downloadId ? '_download' : '') + (ext ? '.' + ext : ext); } export type FileURLType = 'photo' | 'thumb' | 'document' | 'stream' | 'download'; diff --git a/src/helpers/formatBytes.ts b/src/helpers/formatBytes.ts index d1f015d0..04db2cac 100644 --- a/src/helpers/formatBytes.ts +++ b/src/helpers/formatBytes.ts @@ -6,14 +6,18 @@ import {i18n, LangPackKey} from '../lib/langPack'; -export default function formatBytes(bytes: number, decimals = 2) { +export default function formatBytes(bytes: number, decimals: number | 'auto' = 'auto') { if(bytes === 0) return i18n('FileSize.B', [0]); - const k = 1024; - const dm = decimals < 0 ? 0 : decimals; - const sizes: LangPackKey[] = ['FileSize.B', 'FileSize.KB', 'FileSize.MB', 'FileSize.GB']; + const strictDecimals = decimals === 'auto'; + const k = 1024; const i = Math.floor(Math.log(bytes) / Math.log(k)); + const _decimals = decimals === 'auto' ? Math.max(0, i - 1) : decimals; + + const dm = Math.max(0, _decimals); + const sizes: LangPackKey[] = ['FileSize.B', 'FileSize.KB', 'FileSize.MB', 'FileSize.GB']; - return i18n(sizes[i], [parseFloat((bytes / Math.pow(k, i)).toFixed(dm))]); + const fixed = (bytes / Math.pow(k, i)).toFixed(dm); + return i18n(sizes[i], [strictDecimals ? fixed : parseFloat(fixed)]); } diff --git a/src/lib/appManagers/appDocsManager.ts b/src/lib/appManagers/appDocsManager.ts index 5046dd20..524bbfbb 100644 --- a/src/lib/appManagers/appDocsManager.ts +++ b/src/lib/appManagers/appDocsManager.ts @@ -9,7 +9,7 @@ * https://github.com/zhukov/webogram/blob/master/LICENSE */ -import {AccountWallPapers, Document, MessagesSavedGifs, PhotoSize, WallPaper} from '../../layer'; +import {AccountWallPapers, Document, DocumentAttribute, MessagesSavedGifs, PhotoSize, WallPaper} from '../../layer'; import {ReferenceContext} from '../mtproto/referenceDatabase'; import {getFullDate} from '../../helpers/date'; import isObject from '../../helpers/object/isObject'; @@ -23,6 +23,7 @@ import MTProtoMessagePort from '../mtproto/mtprotoMessagePort'; import getDocumentInput from './utils/docs/getDocumentInput'; import getDocumentURL from './utils/docs/getDocumentURL'; import type {ThumbCache} from '../storages/thumbs'; +import makeError from '../../helpers/makeError'; export type MyDocument = Document.document; @@ -216,7 +217,10 @@ export class AppDocsManager extends AppManager { if(doc.type === 'voice' || doc.type === 'round') { // browser will identify extension - doc.file_name = doc.type + '_' + getFullDate(new Date(doc.date * 1000), {monthAsNumber: true, leadingZero: true}).replace(/[:\.]/g, '-').replace(', ', '_'); + const attribute = doc.attributes.find((attribute) => attribute._ === 'documentAttributeFilename') as DocumentAttribute.documentAttributeFilename; + const ext = attribute && attribute.file_name.split('.').pop(); + const date = getFullDate(new Date(doc.date * 1000), {monthAsNumber: true, leadingZero: true}).replace(/[:\.]/g, '-').replace(', ', '_'); + doc.file_name = `${doc.type}_${date}${ext ? '.' + ext : ''}`; } if(isServiceWorkerOnline()) { @@ -400,6 +404,7 @@ export class AppDocsManager extends AppManager { public requestDocPart(docId: DocId, dcId: number, offset: number, limit: number) { const doc = this.getDoc(docId); + if(!doc) return Promise.reject(makeError('NO_DOC')); return this.apiFileManager.requestFilePart(dcId, getDocumentInput(doc), offset, limit); } } diff --git a/src/lib/appManagers/appDownloadManager.ts b/src/lib/appManagers/appDownloadManager.ts index 0e055a3b..990d02d4 100644 --- a/src/lib/appManagers/appDownloadManager.ts +++ b/src/lib/appManagers/appDownloadManager.ts @@ -16,8 +16,7 @@ import noop from '../../helpers/noop'; import getDownloadMediaDetails from './utils/download/getDownloadMediaDetails'; import getDownloadFileNameFromOptions from './utils/download/getDownloadFileNameFromOptions'; import indexOfAndSplice from '../../helpers/array/indexOfAndSplice'; -import {MAX_FILE_SAVE_SIZE} from '../mtproto/mtproto_config'; -import createDownloadAnchor from '../../helpers/dom/createDownloadAnchor'; +import makeError from '../../helpers/makeError'; export type ResponseMethodBlob = 'blob'; export type ResponseMethodJson = 'json'; @@ -38,6 +37,7 @@ type DownloadType = 'url' | 'blob' | 'void' | 'disc'; export class AppDownloadManager { private downloads: {[fileName: string]: {main: Download} & {[type in DownloadType]?: Download}} = {}; + // private downloadsToDisc: {[fileName: string]: Download} = {}; private progress: {[fileName: string]: Progress} = {}; // private progressCallbacks: {[fileName: string]: Array} = {}; private managers: AppManagers; @@ -45,15 +45,14 @@ export class AppDownloadManager { public construct(managers: AppManagers) { this.managers = managers; rootScope.addEventListener('download_progress', (details) => { - this.progress[details.fileName] = details; - // const callbacks = this.progressCallbacks[details.fileName]; // if(callbacks) { // callbacks.forEach((callback) => callback(details)); // } const download = this.downloads[details.fileName]; - if(download) { + if(download?.main?.notifyAll) { + this.progress[details.fileName] = details; download.main.notifyAll(details); } }); @@ -69,17 +68,12 @@ export class AppDownloadManager { }; deferred.cancel = () => { - // try { - const error = new Error('Download canceled'); - error.name = 'AbortError'; + const error = makeError('DOWNLOAD_CANCELED'); this.managers.apiFileManager.cancelDownload(fileName); deferred.reject(error); - deferred.cancel = () => {}; - /* } catch(err) { - - } */ + deferred.cancel = noop; }; deferred.catch(() => { @@ -225,51 +219,93 @@ export class AppDownloadManager { // } // } - public downloadToDisc(options: DownloadMediaOptions) { + public downloadToDisc(options: DownloadMediaOptions, justAttach?: boolean) { const media = options.media; const isDocument = media._ === 'document'; if(!isDocument && !options.thumb) { options.thumb = (media as Photo.photo).sizes.slice().pop() as PhotoSize.photoSize; } - const {downloadOptions, fileName} = getDownloadMediaDetails(options); - if(downloadOptions.size && downloadOptions.size > MAX_FILE_SAVE_SIZE) { - const id = '' + (Math.random() * 0x7FFFFFFF | 0); - const url = `/download/${id}`; - options.downloadId = id; - - const promise = this.downloadMedia(options, 'disc'); - - let iframe: HTMLIFrameElement; - const onProgress = () => { - iframe = document.createElement('iframe'); - iframe.hidden = true; - // iframe.src = sw.scope + fileName; - iframe.src = url; - document.body.append(iframe); - - indexOfAndSplice(promise.listeners, onProgress); - }; - - promise.addNotifyListener(onProgress); - promise.catch(noop).finally(() => { - setTimeout(() => { - iframe?.remove(); - }, 1000); - }); - - return promise; - } else { - const promise = this.downloadMedia(options, 'blob'); - promise.then((blob) => { - const url = URL.createObjectURL(blob); - createDownloadAnchor(url, downloadOptions.fileName || fileName, () => { - URL.revokeObjectURL(url); - }); - }); + // const {fileName: cacheFileName} = getDownloadMediaDetails(options); + // if(justAttach) { + // const promise = this.downloadsToDisc[cacheFileName]; + // if(promise) { + // return promise; + // } + // } + + // const {downloadOptions, fileName} = getDownloadMediaDetails(options); + // if(downloadOptions.size && downloadOptions.size > MAX_FILE_SAVE_SIZE) { + const id = '' + (Math.random() * 0x7FFFFFFF | 0); + // const id = 'test'; + const url = `/download/${id}`; + options.downloadId = id; + + const promise = this.downloadMedia(options, 'disc'); + // this.downloadsToDisc[cacheFileName] = promise; + + if(justAttach) { return promise; } + const iframe = document.createElement('iframe'); + iframe.hidden = true; + iframe.src = url; + document.body.append(iframe); + // createDownloadAnchor(url, 'asd.txt'); + + // const events = [ + // 'emptied', + // 'abort', + // 'suspend', + // 'reset', + // 'error', + // 'ended', + // 'load' + // ].forEach((event) => { + // iframe.addEventListener(event, () => alert(event)); + // iframe.contentWindow.addEventListener(event, () => alert(event)); + // }); + + let element: HTMLElement, hadProgress = false; + const onProgress = () => { + if(hadProgress) { + return; + } + + hadProgress = true; + element = iframe; + + indexOfAndSplice(promise.listeners, onProgress); + }; + + promise.addNotifyListener(onProgress); + promise.catch(noop).finally(() => { + if(!hadProgress) { + onProgress(); + } + + setTimeout(() => { + element?.remove(); + }, 1000); + + // if(this.downloadsToDisc[cacheFileName] === promise) { + // delete this.downloadsToDisc[cacheFileName]; + // } + }); + + return promise; + // } else { + // const promise = this.downloadMedia(options, 'blob'); + // promise.then((blob) => { + // const url = URL.createObjectURL(blob); + // createDownloadAnchor(url, downloadOptions.fileName || fileName, () => { + // URL.revokeObjectURL(url); + // }); + // }); + // return promise; + // } + // const promise = this.downloadMedia(options); // promise.then((blob) => { // const url = URL.createObjectURL(blob); diff --git a/src/lib/appManagers/utils/download/getDownloadFileNameFromOptions.ts b/src/lib/appManagers/utils/download/getDownloadFileNameFromOptions.ts index 6770674c..84560e1c 100644 --- a/src/lib/appManagers/utils/download/getDownloadFileNameFromOptions.ts +++ b/src/lib/appManagers/utils/download/getDownloadFileNameFromOptions.ts @@ -8,5 +8,5 @@ import {getFileNameByLocation} from '../../../../helpers/fileName'; import {DownloadOptions} from '../../../mtproto/apiFileManager'; export default function getDownloadFileNameFromOptions(options: DownloadOptions) { - return getFileNameByLocation(options.location, {fileName: options.fileName}); + return getFileNameByLocation(options.location, options); } diff --git a/src/lib/appManagers/utils/webDocs/getWebDocumentDownloadOptions.ts b/src/lib/appManagers/utils/webDocs/getWebDocumentDownloadOptions.ts index 674ac45e..664e5d16 100644 --- a/src/lib/appManagers/utils/webDocs/getWebDocumentDownloadOptions.ts +++ b/src/lib/appManagers/utils/webDocs/getWebDocumentDownloadOptions.ts @@ -3,7 +3,7 @@ import {DownloadOptions} from '../../../mtproto/apiFileManager'; export default function getWebDocumentDownloadOptions(webDocument: WebDocument): DownloadOptions { return { - dcId: 4, + dcId: 0, location: { _: 'inputWebFileLocation', access_hash: (webDocument as WebDocument.webDocument).access_hash, diff --git a/src/lib/files/cacheStorage.ts b/src/lib/files/cacheStorage.ts index 07c045aa..c556fff8 100644 --- a/src/lib/files/cacheStorage.ts +++ b/src/lib/files/cacheStorage.ts @@ -10,6 +10,7 @@ import MemoryWriter from './memoryWriter'; import FileManager from './memoryWriter'; import FileStorage from './fileStorage'; import makeError from '../../helpers/makeError'; +import deferredPromise from '../../helpers/cancellablePromise'; export type CacheStorageDbName = 'cachedFiles' | 'cachedStreamChunks' | 'cachedAssets'; @@ -124,12 +125,17 @@ export default class CacheStorageController implements FileStorage { }); } - public getWriter(fileName: string, fileSize: number, mimeType: string) { - const writer = new MemoryWriter(mimeType, fileSize, (blob) => { - return this.saveFile(fileName, blob).catch(() => blob); - }); + public prepareWriting(fileName: string, fileSize: number, mimeType: string) { + return { + deferred: deferredPromise(), + getWriter: () => { + const writer = new MemoryWriter(mimeType, fileSize, (blob) => { + return this.saveFile(fileName, blob).catch(() => blob); + }); - return Promise.resolve(writer); + return writer; + } + }; } public static toggleStorage(enabled: boolean, clearWrite: boolean) { diff --git a/src/lib/files/downloadStorage.ts b/src/lib/files/downloadStorage.ts new file mode 100644 index 00000000..7fbf0f4a --- /dev/null +++ b/src/lib/files/downloadStorage.ts @@ -0,0 +1,55 @@ +/* + * https://github.com/morethanwords/tweb + * Copyright (C) 2019-2021 Eduard Kuzmenko + * https://github.com/morethanwords/tweb/blob/master/LICENSE + */ + +import deferredPromise from '../../helpers/cancellablePromise'; +import makeError from '../../helpers/makeError'; +import fileNameRFC from '../../helpers/string/fileNameRFC'; +import {getServiceMessagePort} from '../mtproto/mtproto.worker'; +import DownloadWriter from './downloadWriter'; +import FileStorage from './fileStorage'; +import StreamWriter from './streamWriter'; + +export default class DownloadStorage implements FileStorage { + public getFile(fileName: string): Promise { + return Promise.reject(makeError('NO_ENTRY_FOUND')); + } + + public prepareWriting({fileName, downloadId, size}: { + fileName: string, + downloadId: string, + size: number + }) { + const headers = { + 'Content-Type': 'application/octet-stream; charset=utf-8', + 'Content-Disposition': 'attachment; filename*=UTF-8\'\'' + fileNameRFC(fileName), + ...(size ? {'Content-Length': size} : {}) + }; + + const serviceMessagePort = getServiceMessagePort(); + const promise = serviceMessagePort.invoke('download', { + headers, + id: downloadId + }); + + const deferred = deferredPromise(); + deferred.cancel = () => { + deferred.reject(makeError('DOWNLOAD_CANCELED')); + }; + + deferred.catch(() => { + getServiceMessagePort().invoke('downloadCancel', downloadId); + }); + + promise.then(deferred.resolve, deferred.reject); + + return { + deferred, + getWriter: () => { + return new DownloadWriter(serviceMessagePort, downloadId); + } + }; + } +} diff --git a/src/lib/files/downloadWriter.ts b/src/lib/files/downloadWriter.ts new file mode 100644 index 00000000..f0a0c719 --- /dev/null +++ b/src/lib/files/downloadWriter.ts @@ -0,0 +1,29 @@ +/* + * https://github.com/morethanwords/tweb + * Copyright (C) 2019-2021 Eduard Kuzmenko + * https://github.com/morethanwords/tweb/blob/master/LICENSE + */ + +import {getServiceMessagePort} from '../mtproto/mtproto.worker'; +import ServiceMessagePort from '../serviceWorker/serviceMessagePort'; +import StreamWriter from './streamWriter'; + +export default class DownloadWriter implements StreamWriter { + constructor( + private serviceMessagePort: ServiceMessagePort, + private downloadId: string + ) { + this.serviceMessagePort = getServiceMessagePort(); + } + + public async write(part: Uint8Array, offset?: number) { + return this.serviceMessagePort.invoke('downloadChunk', { + id: this.downloadId, + chunk: part + }); + } + + public finalize(saveToStorage?: boolean): Promise { + return this.serviceMessagePort.invoke('downloadFinalize', this.downloadId).then(() => undefined); + } +} diff --git a/src/lib/files/fileStorage.ts b/src/lib/files/fileStorage.ts index 69eff3a5..b882bf39 100644 --- a/src/lib/files/fileStorage.ts +++ b/src/lib/files/fileStorage.ts @@ -4,10 +4,10 @@ * https://github.com/morethanwords/tweb/blob/master/LICENSE */ +import {CancellablePromise} from '../../helpers/cancellablePromise'; import StreamWriter from './streamWriter'; export default abstract class FileStorage { public abstract getFile(fileName: string): Promise; - - public abstract getWriter(fileName: string, fileSize: number, mimeType: string): Promise; + public abstract prepareWriting(...args: any[]): {deferred: CancellablePromise, getWriter: () => StreamWriter}; } diff --git a/src/lib/mtproto/apiFileManager.ts b/src/lib/mtproto/apiFileManager.ts index d2e2b7fb..d85b9930 100644 --- a/src/lib/mtproto/apiFileManager.ts +++ b/src/lib/mtproto/apiFileManager.ts @@ -18,7 +18,6 @@ import {DcId} from '../../types'; import CacheStorageController from '../files/cacheStorage'; import {logger, LogTypes} from '../logger'; import assumeType from '../../helpers/assumeType'; -import ctx from '../../environment/ctx'; import noop from '../../helpers/noop'; import readBlobAsArrayBuffer from '../../helpers/blob/readBlobAsArrayBuffer'; import bytesToHex from '../../helpers/bytes/bytesToHex'; @@ -32,13 +31,15 @@ import type {Progress} from '../appManagers/appDownloadManager'; import getDownloadMediaDetails from '../appManagers/utils/download/getDownloadMediaDetails'; import networkStats from './networkStats'; import getDownloadFileNameFromOptions from '../appManagers/utils/download/getDownloadFileNameFromOptions'; -import {getServiceMessagePort} from './mtproto.worker'; import StreamWriter from '../files/streamWriter'; import FileStorage from '../files/fileStorage'; -import fileNameRFC from '../../helpers/string/fileNameRFC'; import {MAX_FILE_SAVE_SIZE} from './mtproto_config'; import throttle from '../../helpers/schedulers/throttle'; import makeError from '../../helpers/makeError'; +import readBlobAsUint8Array from '../../helpers/blob/readBlobAsUint8Array'; +import DownloadStorage from '../files/downloadStorage'; +import copy from '../../helpers/object/copy'; +import indexOfAndSplice from '../../helpers/array/indexOfAndSplice'; type Delayed = { offset: number, @@ -82,10 +83,13 @@ export type MyUploadFile = UploadFile.uploadFile | UploadWebFile.uploadWebFile; // originalPayload: ReferenceBytes // }; -const MAX_FILE_PART_SIZE = 1 * 1024 * 1024; +const MAX_DOWNLOAD_FILE_PART_SIZE = 1 * 1024 * 1024; +const MAX_UPLOAD_FILE_PART_SIZE = 512 * 1024; +const MIN_PART_SIZE = 128 * 1024; +const AVG_PART_SIZE = 512 * 1024; -const REGULAR_DOWNLOAD_DELTA = 36; -const PREMIUM_DOWNLOAD_DELTA = 72; +const REGULAR_DOWNLOAD_DELTA = (9 * 512 * 1024) / MIN_PART_SIZE; +const PREMIUM_DOWNLOAD_DELTA = REGULAR_DOWNLOAD_DELTA * 2; const IGNORE_ERRORS: Set = new Set([ 'DOWNLOAD_CANCELED', @@ -96,11 +100,16 @@ const IGNORE_ERRORS: Set = new Set([ export class ApiFileManager extends AppManager { private cacheStorage = new CacheStorageController('cachedFiles'); + private downloadStorage = new DownloadStorage(); private downloadPromises: { [fileName: string]: DownloadPromise } = {}; + // private downloadToDiscPromises: { + // [fileName: string]: DownloadPromise + // } = {}; + private uploadPromises: { [fileName: string]: CancellablePromise } = {}; @@ -133,6 +142,7 @@ export class ApiFileManager extends AppManager { private maxUploadParts = 4000; private maxDownloadParts = 8000; + private webFileDcId: DcId; protected after() { setInterval(() => { // clear old promises @@ -144,6 +154,10 @@ export class ApiFileManager extends AppManager { } }, 1800e3); + this.rootScope.addEventListener('config', (config) => { + this.webFileDcId = config.webfile_dc_id; + }); + this.rootScope.addEventListener('app_config', (appConfig) => { this.maxUploadParts = this.rootScope.premium ? appConfig.upload_max_fileparts_premium : appConfig.upload_max_fileparts_default; this.maxDownloadParts = appConfig.upload_max_fileparts_premium; @@ -173,7 +187,7 @@ export class ApiFileManager extends AppManager { private downloadCheck(dcId: string | number) { const downloadPull = this.downloadPulls[dcId]; - const downloadLimit = dcId === 'upload' ? 24 : (this.rootScope.premium ? PREMIUM_DOWNLOAD_DELTA : REGULAR_DOWNLOAD_DELTA); + const downloadLimit = /* dcId === 'upload' ? 24 : */(this.rootScope.premium ? PREMIUM_DOWNLOAD_DELTA : REGULAR_DOWNLOAD_DELTA); // const downloadLimit = Infinity; if(this.downloadActives[dcId] >= downloadLimit || !downloadPull?.length) { @@ -187,7 +201,7 @@ export class ApiFileManager extends AppManager { this.downloadActives[dcId] += activeDelta; const promise = data.cb(); - const networkPromise = networkStats.waitForChunk(dcId as DcId, activeDelta * 1024 * 128); + const networkPromise = networkStats.waitForChunk(dcId as DcId, activeDelta * MIN_PART_SIZE); Promise.race([ promise, networkPromise @@ -235,7 +249,7 @@ export class ApiFileManager extends AppManager { public requestWebFilePart(dcId: DcId, location: InputWebFileLocation, offset: number, limit: number, id = 0, queueId = 0, checkCancel?: () => void) { return this.downloadRequest(dcId, id, async() => { // do not remove async, because checkCancel will throw an error - checkCancel && checkCancel(); + checkCancel?.(); return this.apiManager.invokeApi('upload.getWebFile', { location, @@ -248,17 +262,26 @@ export class ApiFileManager extends AppManager { }, this.getDelta(limit), queueId); } - public requestFilePart(dcId: DcId, location: InputFileLocation, offset: number, limit: number, id = 0, queueId = 0, checkCancel?: () => void) { + public requestFilePart( + dcId: DcId, + location: InputFileLocation, + offset: number, + limit: number, + id = 0, + queueId = 0, + checkCancel?: () => void + ) { return this.downloadRequest(dcId, id, async() => { // do not remove async, because checkCancel will throw an error - checkCancel && checkCancel(); + checkCancel?.(); const invoke = async(): Promise => { - checkCancel && checkCancel(); // do not remove async, because checkCancel will throw an error + checkCancel?.(); // do not remove async, because checkCancel will throw an error // * IMPORTANT: reference can be changed in previous request const reference = (location as InputFileLocation.inputDocumentFileLocation).file_reference?.slice(); - const promise = /* pause(1000).then(() => */this.apiManager.invokeApi('upload.getFile', { + const promise = // pause(offset > (100 * 1024 * 1024) ? 10000000 : 0).then(() => + this.apiManager.invokeApi('upload.getFile', { location, offset, limit @@ -268,6 +291,8 @@ export class ApiFileManager extends AppManager { }) as Promise/* ) */; return promise.catch((err: ApiError) => { + checkCancel?.(); + if(err.type === 'FILE_REFERENCE_EXPIRED') { return this.refreshReference(location as InputFileLocation.inputDocumentFileLocation, reference).then(invoke); } @@ -295,19 +320,22 @@ export class ApiFileManager extends AppManager { } */ private getDelta(bytes: number) { - return bytes / 1024 / 128; + return bytes / MIN_PART_SIZE; } private getLimitPart(size: number, isUpload: boolean): number { if(!size) { // * sometimes size can be 0 (e.g. avatars, webDocuments) - return 512 * 1024; + return AVG_PART_SIZE; } - let bytes = 128 * 1024; + // return 1 * 1024 * 1024; + + let bytes = MIN_PART_SIZE; const maxParts = isUpload ? this.maxUploadParts : this.maxDownloadParts; + const maxPartSize = isUpload ? MAX_UPLOAD_FILE_PART_SIZE : MAX_DOWNLOAD_FILE_PART_SIZE; // usually it will stick to 512Kb size if the file is too big - while((size / bytes) > maxParts && bytes < MAX_FILE_PART_SIZE) { + while((size / bytes) > maxParts && bytes < maxPartSize) { bytes *= 2; } /* if(size < 1e6 || !size) bytes = 512; @@ -399,201 +427,230 @@ export class ApiFileManager extends AppManager { return this.uploadPromises[fileName]; } - public download(options: DownloadOptions): DownloadPromise { - const size = options.size ?? 0; - const {dcId, location, downloadId} = options; - + private getConvertMethod(mimeType: string) { let process: ApiFileManager['uncompressTGS'] | ApiFileManager['convertWebp']; - - if(downloadId) { - - } else if(options.mimeType === 'application/x-tgwallpattern') { + if(mimeType === 'application/x-tgwallpattern') { process = this.uncompressTGV; - options.mimeType = 'image/svg+xml'; - } else if(options.mimeType === 'image/webp' && !getEnvironment().IS_WEBP_SUPPORTED) { + mimeType = 'image/svg+xml'; + } else if(mimeType === 'image/webp' && !getEnvironment().IS_WEBP_SUPPORTED) { process = this.convertWebp; - options.mimeType = 'image/png'; - } else if(options.mimeType === 'application/x-tgsticker') { + mimeType = 'image/png'; + } else if(mimeType === 'application/x-tgsticker') { process = this.uncompressTGS; - options.mimeType = 'application/json'; - } else if(options.mimeType === 'audio/ogg' && !getEnvironment().IS_OPUS_SUPPORTED) { + mimeType = 'application/json'; + } else if(mimeType === 'audio/ogg' && !getEnvironment().IS_OPUS_SUPPORTED) { process = this.convertOpus; - options.mimeType = 'audio/wav'; + mimeType = 'audio/wav'; } - const fileName = getDownloadFileNameFromOptions(options); - const cachedPromise = options.downloadId ? undefined : this.downloadPromises[fileName]; - let fileStorage: FileStorage = this.getFileStorage(); + return {mimeType, process}; + } - this.debug && this.log('downloadFile', fileName, size, location, options.mimeType); + private allocateDeferredPromises(startOffset: number, size: number, limitPart: number) { + const delayed: Delayed[] = []; + let offset = startOffset; + let writePromise: CancellablePromise = Promise.resolve(), + writeDeferred: CancellablePromise; + do { + writeDeferred = deferredPromise(); + delayed.push({offset, writePromise, writeDeferred}); + writePromise = writeDeferred; + offset += limitPart; + } while(offset < size); + + return delayed; + } - /* if(options.queueId) { - this.log.error('downloadFile queueId:', fileName, options.queueId); - } */ + public download(options: DownloadOptions): DownloadPromise { + const size = options.size ?? 0; + const {dcId, location, downloadId} = options; - if(cachedPromise) { - // this.log('downloadFile cachedPromise'); + const originalMimeType = options.mimeType; + const convertMethod = this.getConvertMethod(originalMimeType); + const {process} = convertMethod; + options.mimeType = convertMethod.mimeType || 'image/jpeg'; - if(size) { - return cachedPromise.then((blob) => { - if(blob instanceof Blob && blob.size < size) { - this.debug && this.log('downloadFile need to deleteFile, wrong size:', blob.size, size); + const fileName = getDownloadFileNameFromOptions(options); + const cacheFileName = downloadId ? getDownloadFileNameFromOptions({...copy(options), downloadId: undefined}) : fileName; + const cacheStorage: FileStorage = this.getFileStorage(); + const downloadStorage: FileStorage = downloadId ? this.downloadStorage : undefined; + let deferred: DownloadPromise = downloadId ? undefined : this.downloadPromises[fileName]; - return this.delete(fileName).then(() => { - return this.download(options); - }).catch(() => { - return this.download(options); - }); - } else { - return blob; - } - }); - } else { - return cachedPromise; - } + this.debug && this.log('downloadFile', fileName, options); + + if(deferred) { + return deferred; } - const deferred: DownloadPromise = deferredPromise(); - const mimeType = options.mimeType || 'image/jpeg'; + // if(deferred) { + // if(size) { + // return deferred.then(async(blob) => { + // if(blob instanceof Blob && blob.size < size) { + // this.debug && this.log('downloadFile need to deleteFile, wrong size:', blob.size, size); + + // try { + // await this.delete(fileName); + // } finally { + // return this.download(options); + // } + // } else { + // return blob; + // } + // }); + // } else { + // return deferred; + // } + // } + + const errorHandler = (item: typeof cachePrepared, error: ApiError) => { + if(item?.error) { + return; + } - let error: ApiError; - let resolved = false; - let cacheWriter: StreamWriter; - let errorHandler = (_error: typeof error) => { - error = _error; - delete this.downloadPromises[fileName]; - deferred.reject(error); - errorHandler = () => {}; + for(const p of prepared) { + if(item && item !== p) { + continue; + } - if(cacheWriter && (!error || error.type !== 'DOWNLOAD_CANCELED')) { - cacheWriter.truncate?.(); + p.error = error; + p.deferred.reject(error); } }; const id = this.tempId++; + const limitPart = options.limitPart || this.getLimitPart(size, false); - if(downloadId) { - const headers = { - 'Content-Type': 'application/octet-stream; charset=utf-8', - 'Content-Disposition': 'attachment; filename*=UTF-8\'\'' + fileNameRFC(options.fileName), - // 'Content-Disposition': `attachment; filename="${options.fileName}"`, - // 'Content-Type': 'application/octet-stream; charset=utf-8', - ...(size ? {'Content-Length': size} : {}) - }; + let getFile: FileStorage['getFile'] = cacheStorage.getFile.bind(cacheStorage); - const serviceMessagePort = getServiceMessagePort(); - const promise = serviceMessagePort.invoke('download', { - fileName, - headers, - id: downloadId - }); + let cachePrepared: ReturnType & {writer?: StreamWriter, error?: ApiError}, + downloadPrepared: typeof cachePrepared; + const prepared: (typeof cachePrepared)[] = []; + const possibleSize = size || limitPart; - promise.catch(errorHandler); - deferred.catch(() => { - getServiceMessagePort().invoke('downloadCancel', downloadId); - }); + const getErrorsCount = () => prepared.reduce((acc, item) => acc + +!!item.error, 0); - class f implements StreamWriter { - constructor() { + const attach = (item: typeof cachePrepared, fileName: string) => { + const {deferred} = item; + const _errorHandler = errorHandler.bind(null, item); + deferred.cancel = () => deferred.reject(makeError('DOWNLOAD_CANCELED')); + deferred.catch((error) => { + _errorHandler(error); + item.writer?.truncate?.(); + }).finally(() => { + if(this.downloadPromises[fileName] === deferred) { + delete this.downloadPromises[fileName]; } - public async write(part: Uint8Array, offset?: number) { - return serviceMessagePort.invoke('downloadChunk', { - id: downloadId, - chunk: part - }); - } + delete item.writer; + indexOfAndSplice(prepared, item); + }); - public finalize(saveToStorage?: boolean): Promise { - return serviceMessagePort.invoke('downloadFinalize', downloadId).then(() => null); - } - } + this.downloadPromises[fileName] = deferred; - class d implements FileStorage { - public getFile(fileName: string): Promise { - return Promise.reject(); - } + prepared.push(item); + }; - public getWriter(fileName: string, fileSize: number, mimeType: string): Promise { - return Promise.resolve(new f()); - } + if(cacheStorage && (!downloadStorage || possibleSize <= MAX_FILE_SAVE_SIZE)) { + cachePrepared = cacheStorage.prepareWriting(cacheFileName, possibleSize, options.mimeType) + attach(cachePrepared, cacheFileName); + } + + if(downloadStorage) { + downloadPrepared = downloadStorage.prepareWriting({ + fileName: options.fileName, // it's doc file_name + downloadId, + size: possibleSize + }); + attach(downloadPrepared, fileName); + + if(cachePrepared) { // cancel cache too + downloadPrepared.deferred.catch((err) => cachePrepared.deferred.reject(err)); } - fileStorage = new d(); + // this.downloadToDiscPromises[cacheFileName] = deferred; + // deferred.catch(noop).finally(() => { + // if(this.downloadToDiscPromises[cacheFileName] === deferred) { + // delete this.downloadToDiscPromises[cacheFileName]; + // } + // }); } - fileStorage.getFile(fileName).then(async(blob: Blob) => { - // throw ''; + deferred = downloadPrepared?.deferred ?? cachePrepared.deferred; - if(blob.size < size) { - if(!options.onlyCache) { - await this.delete(fileName); - } + if(downloadStorage && process) { // then have to load file again + getFile = downloadStorage.getFile.bind(downloadStorage); + } + + getFile(cacheFileName).then(async(blob: Blob) => { + checkCancel(); + + // if(blob.size < size) { + // if(!options.onlyCache) { + // await this.delete(cacheFileName); + // checkCancel(); + // } - throw false; + // throw makeError('NO_ENTRY_FOUND'); + // } + + if(downloadPrepared) { + const writer = downloadPrepared.writer = downloadPrepared.getWriter(); + checkCancel(); + + const arr = await readBlobAsUint8Array(blob); + checkCancel(); + await writer.write(arr); + checkCancel(); + + downloadPrepared.deferred.resolve(await writer.finalize()); } - deferred.resolve(blob); + if(cachePrepared) { + cachePrepared.deferred.resolve(blob); + } }).catch(async(err: ApiError) => { if(options.onlyCache) { - errorHandler(err); + errorHandler(null, err); return; } - // this.log('not cached', fileName); - const limit = options.limitPart || this.getLimitPart(size, false); - const writerPromise = fileStorage.getWriter(fileName, size || limit, mimeType); - - const writer = cacheWriter = await writerPromise; + prepared.forEach((p) => { + p.writer = p.getWriter(); + }); - let offset: number; - const startOffset = 0; - let writePromise: CancellablePromise = Promise.resolve(), - writeDeferred: CancellablePromise; - // const maxRequests = 13107200 / limit; // * 100 Mb speed const maxRequests = Infinity; - const processDownloaded = async(bytes: Uint8Array) => { - if(process) { - // const perf = performance.now(); - const processed = await process(bytes, fileName); - // this.log('downloadFile process downloaded time', performance.now() - perf, mimeType, process); - return processed; - } + const isWebFile = location._ === 'inputWebFileLocation'; + const requestPart = (isWebFile ? this.requestWebFilePart : this.requestFilePart).bind(this); - return bytes; - }; - - const r = location._ === 'inputWebFileLocation' ? this.requestWebFilePart.bind(this) : this.requestFilePart.bind(this); + if(isWebFile && this.webFileDcId === undefined) { + await this.apiManager.getConfig(); + checkCancel(); + } - const delayed: Delayed[] = []; - offset = startOffset; - do { - writeDeferred = deferredPromise(); - delayed.push({offset, writePromise, writeDeferred}); - writePromise = writeDeferred; - offset += limit; - } while(offset < size); + const delayed = this.allocateDeferredPromises(0, size, limitPart); - const progress: Progress = {done: 0, offset, total: size, fileName}; + const progress: Progress = {done: 0, offset: 0, total: size, fileName}; const dispatchProgress = () => { - progress.done = done; - deferred.notify?.(progress); + try { + checkCancel(); + progress.done = done; + this.rootScope.dispatchEvent('download_progress', progress); + } catch(err) {} }; const throttledDispatchProgress = throttle(dispatchProgress, 50, true); let done = 0; const superpuper = async() => { - // if(!delayed.length) return; - const {offset, writePromise, writeDeferred} = delayed.shift(); try { checkCancel(); - // @ts-ignore - const result = await r(dcId, location as any, offset, limit, id, options.queueId, checkCancel); + const requestPerf = performance.now(); + const result = await requestPart(dcId, location as any, offset, limitPart, id, options.queueId, checkCancel); + const requestTime = performance.now() - requestPerf; const bytes = result.bytes; @@ -603,7 +660,7 @@ export class ApiFileManager extends AppManager { const byteLength = bytes.byteLength; this.debug && this.log('downloadFile requestFilePart result:', fileName, result); - const isFinal = (offset + limit) >= size || !byteLength; + const isFinal = (offset + limitPart) >= size || !byteLength; if(byteLength) { done += byteLength; @@ -613,68 +670,65 @@ export class ApiFileManager extends AppManager { throttledDispatchProgress(); } + const writeQueuePerf = performance.now(); await writePromise; checkCancel(); + const writeQueueTime = performance.now() - writeQueuePerf; - // const perf = performance.now(); - await writer.write(bytes, offset); + const perf = performance.now(); + await Promise.all(prepared.map(({writer}) => writer?.write(bytes, offset))); checkCancel(); - // downloadId && this.log('write time', performance.now() - perf); + // downloadId && this.log('write time', performance.now() - perf, 'request time', requestTime, 'queue time', writeQueueTime); } if(isFinal && process) { - const bytes = writer.getParts(); - const processedResult = await processDownloaded(bytes); - checkCancel(); + const promises = prepared + .filter(({writer}) => writer?.getParts && writer.replaceParts) + .map(async({writer}) => { + const bytes = writer.getParts(); + const processedResult = await process(bytes, cacheFileName); + writer.replaceParts(processedResult); + }); - writer.replaceParts(processedResult); + await Promise.all(promises); + checkCancel(); } writeDeferred.resolve(); if(isFinal) { - resolved = true; - const realSize = size || byteLength; - if(!size) { - writer.trim(realSize); + if(!size || byteLength < size) { + prepared.forEach(({writer}) => writer?.trim?.(realSize)); } - deferred.resolve(await writer.finalize(realSize <= MAX_FILE_SAVE_SIZE)); + const saveToStorage = realSize <= MAX_FILE_SAVE_SIZE; + prepared.forEach((item) => { + const {deferred, writer} = item; + if(deferred.isFulfilled || deferred.isRejected || !writer) { + return; + } + + const result = writer.finalize(saveToStorage); + deferred.resolve(result); + }); } } catch(err) { - errorHandler(err as ApiError); + errorHandler(null, err as ApiError); } }; for(let i = 0, length = Math.min(maxRequests, delayed.length); i < length; ++i) { superpuper(); } - }); + }).catch(noop); const checkCancel = () => { - if(error) { - throw error; + if(getErrorsCount() === prepared.length) { + throw prepared[0].error; } }; - deferred.cancel = () => { - if(!error && !resolved) { - const error = makeError('DOWNLOAD_CANCELED'); - errorHandler(error); - } - }; - - deferred.notify = (progress: Progress) => { - this.rootScope.dispatchEvent('download_progress', progress); - }; - - this.downloadPromises[fileName] = deferred; - - deferred.catch(noop).finally(() => { - delete this.downloadPromises[fileName]; - }); - return deferred; } @@ -736,23 +790,14 @@ export class ApiFileManager extends AppManager { } public upload({file, fileName}: {file: Blob | File, fileName?: string}) { - const fileSize = file.size, - isBigFile = fileSize >= 10485760; - - let canceled = false, - resolved = false, - doneParts = 0; - const partSize = this.getLimitPart(fileSize, true); - fileName ||= getFileNameForUpload(file); + const fileSize = file.size; + const isBigFile = fileSize >= 10485760; + const partSize = this.getLimitPart(fileSize, true); const activeDelta = this.getDelta(partSize); - const totalParts = Math.ceil(fileSize / partSize); const fileId = randomLong(); - - let _part = 0; - const resultInputFile: InputFile = { _: isBigFile ? 'inputFileBig' : 'inputFile', id: fileId as any, @@ -767,6 +812,7 @@ export class ApiFileManager extends AppManager { return deferred; } + let canceled = false, resolved = false; let errorHandler = (error: ApiError) => { if(error?.type !== 'UPLOAD_CANCELED') { this.log.error('Up Error', error); @@ -774,74 +820,47 @@ export class ApiFileManager extends AppManager { deferred.reject(error); canceled = true; - errorHandler = () => {}; + errorHandler = noop; }; const method = isBigFile ? 'upload.saveBigFilePart' : 'upload.saveFilePart'; - const id = this.tempId++; const self = this; function* generator() { + let _part = 0, doneParts = 0; for(let offset = 0; offset < fileSize; offset += partSize) { const part = _part++; // 0, 1 - yield self.downloadRequest('upload', id, () => { + yield self.downloadRequest('upload', id, async() => { + checkCancel(); + const blob = file.slice(offset, offset + partSize); + const buffer = await readBlobAsArrayBuffer(blob); + checkCancel(); - return readBlobAsArrayBuffer(blob).then((buffer) => { + self.debug && self.log('Upload file part, isBig:', isBigFile, part, buffer.byteLength, new Uint8Array(buffer).length, new Uint8Array(buffer).slice().length); + + return self.apiManager.invokeApi(method, { + file_id: fileId, + file_part: part, + file_total_parts: totalParts, + bytes: buffer + } as any, { + fileUpload: true + }).then(() => { if(canceled) { - throw makeError('UPLOAD_CANCELED'); + return; } - self.debug && self.log('Upload file part, isBig:', isBigFile, part, buffer.byteLength, new Uint8Array(buffer).length, new Uint8Array(buffer).slice().length); + ++doneParts; + const progress: Progress = {done: doneParts * partSize, offset, total: fileSize, fileName}; + deferred.notify(progress); - /* const u = new Uint8Array(buffer.byteLength); - for(let i = 0; i < u.length; ++i) { - //u[i] = Math.random() * 255 | 0; - u[i] = 0; + if(doneParts >= totalParts) { + deferred.resolve(resultInputFile); + resolved = true; } - buffer = u.buffer; */ - - /* setTimeout(() => { - doneParts++; - uploadResolve(); - - //////this.log('Progress', doneParts * partSize / fileSize); - - self.log('done part', part, doneParts); - - deferred.notify({done: doneParts * partSize, total: fileSize}); - - if(doneParts >= totalParts) { - deferred.resolve(resultInputFile); - resolved = true; - } - }, 1250); - return; */ - - return self.apiManager.invokeApi(method, { - file_id: fileId, - file_part: part, - file_total_parts: totalParts, - bytes: buffer/* new Uint8Array(buffer) */ - } as any, { - // startMaxLength: partSize + 256, - fileUpload: true - }).then(() => { - if(canceled) { - return; - } - - ++doneParts; - const progress: Progress = {done: doneParts * partSize, offset, total: fileSize, fileName}; - deferred.notify(progress); - - if(doneParts >= totalParts) { - deferred.resolve(resultInputFile); - resolved = true; - } - }, errorHandler); - }); + }, errorHandler); }, activeDelta).catch(errorHandler); } } @@ -855,14 +874,16 @@ export class ApiFileManager extends AppManager { }; const maxRequests = Infinity; - // const maxRequests = 10; - /* for(let i = 0; i < 10; ++i) { - process(); - } */ for(let i = 0, length = Math.min(maxRequests, totalParts); i < length; ++i) { process(); } + const checkCancel = () => { + if(canceled) { + throw makeError('UPLOAD_CANCELED'); + } + }; + deferred.cancel = () => { if(!canceled && !resolved) { canceled = true; @@ -875,7 +896,9 @@ export class ApiFileManager extends AppManager { }; deferred.finally(() => { - delete this.uploadPromises[fileName]; + if(this.uploadPromises[fileName] === deferred) { + delete this.uploadPromises[fileName]; + } }); return this.uploadPromises[fileName] = deferred; diff --git a/src/lib/mtproto/api_methods.ts b/src/lib/mtproto/api_methods.ts index 75f016a1..697aed0c 100644 --- a/src/lib/mtproto/api_methods.ts +++ b/src/lib/mtproto/api_methods.ts @@ -6,7 +6,7 @@ import ctx from '../../environment/ctx'; import {ignoreRestrictionReasons} from '../../helpers/restrictions'; -import {MethodDeclMap, User} from '../../layer'; +import {Config, MethodDeclMap, User} from '../../layer'; import {InvokeApiOptions} from '../../types'; import {AppManager} from '../appManagers/manager'; import {MTAppConfig} from './appConfig'; @@ -43,8 +43,8 @@ export default abstract class ApiManagerMethods extends AppManager { } } = {}; + private config: Config; private appConfig: MTAppConfig; - private getAppConfigPromise: Promise; constructor() { super(); @@ -140,7 +140,7 @@ export default abstract class ApiManagerMethods extends AppManager { processResult: (response: MethodDeclMap[T]['res']) => R, processError?: (error: ApiError) => any, params?: MethodDeclMap[T]['req'], - options?: InvokeApiOptions & {cacheKey?: string} + options?: InvokeApiOptions & {cacheKey?: string, overwrite?: boolean} }): Promise> { o.params ??= {}; o.options ??= {}; @@ -154,10 +154,32 @@ export default abstract class ApiManagerMethods extends AppManager { return oldPromise; } + const getNewPromise = () => { + const promise = map.get(cacheKey); + return promise === p ? undefined : promise; + } + const originalPromise = this.invokeApi(method, params, options); - const newPromise: Promise> = originalPromise.then(processResult, processError); + const newPromise: Promise> = originalPromise.then((result) => { + return getNewPromise() || processResult(result); + }, (error) => { + const promise = getNewPromise(); + if(promise) { + return promise; + } + + if(!processError) { + throw error; + } + + return processError(error); + }); const p = newPromise.finally(() => { + if(map.get(cacheKey) !== p) { + return; + } + map.delete(cacheKey); if(!map.size) { delete cache[method]; @@ -226,24 +248,38 @@ export default abstract class ApiManagerMethods extends AppManager { } } - public getConfig() { - return this.invokeApiCacheable('help.getConfig'); + public getConfig(overwrite?: boolean) { + if(this.config && !overwrite) { + return this.config; + } + + return this.invokeApiSingleProcess({ + method: 'help.getConfig', + params: {}, + processResult: (config) => { + this.config = config; + this.rootScope.dispatchEvent('config', config); + return config; + }, + options: {overwrite} + }); } public getAppConfig(overwrite?: boolean) { - if(this.appConfig && !overwrite) return this.appConfig; - if(this.getAppConfigPromise && !overwrite) return this.getAppConfigPromise; - const promise: Promise = this.getAppConfigPromise = this.invokeApi('help.getAppConfig').then((config: MTAppConfig) => { - if(this.getAppConfigPromise !== promise) { - return this.getAppConfigPromise; - } + if(this.appConfig && !overwrite) { + return this.appConfig; + } - this.appConfig = config; - ignoreRestrictionReasons(config.ignore_restriction_reasons ?? []); - this.rootScope.dispatchEvent('app_config', config); - return config; + return this.invokeApiSingleProcess({ + method: 'help.getAppConfig', + params: {}, + processResult: (config: MTAppConfig) => { + this.appConfig = config; + ignoreRestrictionReasons(config.ignore_restriction_reasons ?? []); + this.rootScope.dispatchEvent('app_config', config); + return config; + }, + options: {overwrite} }); - - return promise; } } diff --git a/src/lib/mtproto/dcConfigurator.ts b/src/lib/mtproto/dcConfigurator.ts index b76c9a45..2bb7e1f7 100644 --- a/src/lib/mtproto/dcConfigurator.ts +++ b/src/lib/mtproto/dcConfigurator.ts @@ -51,7 +51,7 @@ export function getTelegramConnectionSuffix(connectionType: ConnectionType) { // #if MTPROTO_HAS_WS export function constructTelegramWebSocketUrl(dcId: DcId, connectionType: ConnectionType, premium?: boolean) { const suffix = getTelegramConnectionSuffix(connectionType); - const path = connectionType !== 'client' ? 'apiws' + (premium ? PREMIUM_SUFFIX : TEST_SUFFIX) : ('apiws' + TEST_SUFFIX); + const path = connectionType !== 'client' ? 'apiws' + TEST_SUFFIX + (premium ? PREMIUM_SUFFIX : '') : ('apiws' + TEST_SUFFIX); const chosenServer = `wss://${App.suffix.toLowerCase()}ws${dcId}${suffix}.web.telegram.org/${path}`; return chosenServer; diff --git a/src/lib/mtproto/transports/websocket.ts b/src/lib/mtproto/transports/websocket.ts index 72b2538b..e0a8c8f8 100644 --- a/src/lib/mtproto/transports/websocket.ts +++ b/src/lib/mtproto/transports/websocket.ts @@ -86,8 +86,8 @@ export default class Socket extends EventListenerBase<{ this.close(); }; - private handleClose = () => { - this.log('closed'/* , event, this.pending, this.ws.bufferedAmount */); + private handleClose = (e?: CloseEvent) => { + this.log('closed', e/* , this.pending, this.ws.bufferedAmount */); this.removeListeners(); this.dispatchEvent('close'); diff --git a/src/lib/rootScope.ts b/src/lib/rootScope.ts index fe02ee4f..9c755f9f 100644 --- a/src/lib/rootScope.ts +++ b/src/lib/rootScope.ts @@ -4,7 +4,7 @@ * https://github.com/morethanwords/tweb/blob/master/LICENSE */ -import type {Message, StickerSet, Update, NotifyPeer, PeerNotifySettings, PollResults, Poll, WebPage, GroupCall, GroupCallParticipant, ReactionCount, MessagePeerReaction, PhoneCall} from '../layer'; +import type {Message, StickerSet, Update, NotifyPeer, PeerNotifySettings, PollResults, Poll, WebPage, GroupCall, GroupCallParticipant, ReactionCount, MessagePeerReaction, PhoneCall, Config} from '../layer'; import type {AppMessagesManager, Dialog, MessagesStorageKey, MyMessage} from './appManagers/appMessagesManager'; import type {MyDialogFilter} from './storages/filters'; import type {Folder} from './storages/dialogs'; @@ -143,6 +143,7 @@ export type BroadcastEvents = { 'premium_toggle': boolean, + 'config': Config, 'app_config': MTAppConfig }; diff --git a/src/lib/serviceWorker/download.ts b/src/lib/serviceWorker/download.ts new file mode 100644 index 00000000..f7d8fba2 --- /dev/null +++ b/src/lib/serviceWorker/download.ts @@ -0,0 +1,162 @@ +/* + * https://github.com/morethanwords/tweb + * Copyright (C) 2019-2021 Eduard Kuzmenko + * https://github.com/morethanwords/tweb/blob/master/LICENSE + */ + +import type {ServiceDownloadTaskPayload} from './serviceMessagePort'; +import type ServiceMessagePort from './serviceMessagePort'; +import deferredPromise, {CancellablePromise} from '../../helpers/cancellablePromise'; +import makeError from '../../helpers/makeError'; +import pause from '../../helpers/schedulers/pause'; + +type DownloadType = Uint8Array; +type DownloadItem = ServiceDownloadTaskPayload & { + // transformStream: TransformStream, + readableStream: ReadableStream, + // writableStream: WritableStream, + // writer: WritableStreamDefaultWriter, + // controller: TransformStreamDefaultController, + controller: ReadableStreamController, + promise: CancellablePromise, + // downloadPromise: Promise, + used?: boolean +}; +const downloadMap: Map = new Map(); +const DOWNLOAD_ERROR = makeError('UNKNOWN'); +const DOWNLOAD_TEST = false; + +type A = Parameters['addMultipleEventsListeners']>[0]; + +const events: A = { + download: (payload) => { + const {id} = payload; + if(downloadMap.has(id)) { + return Promise.reject(DOWNLOAD_ERROR); + } + + // const y = (20 * 1024 * 1024) / payload.limitPart; + // const strategy = new ByteLengthQueuingStrategy({highWaterMark: y}); + // let controller: TransformStreamDefaultController; + const strategy = new CountQueuingStrategy({highWaterMark: 1}); + // const transformStream = new TransformStream(/* { + // start: (_controller) => controller = _controller, + // }, */undefined, strategy, strategy); + + // const {readable, writable} = transformStream; + // const writer = writable.getWriter(); + + const promise = deferredPromise(); + promise.then(() => { + setTimeout(() => { + downloadMap.delete(id); + }, 5e3); + }, () => { + downloadMap.delete(id); + }); + + // writer.closed.then(promise.resolve, promise.reject); + + let controller: ReadableStreamController; + const readable = new ReadableStream({ + start: (_controller) => { + controller = _controller; + }, + + cancel: (reason) => { + promise.reject(DOWNLOAD_ERROR); + } + }, strategy); + + // writer.closed.catch(noop).finally(() => { + // log.error('closed writer'); + // onEnd(); + // }); + + // const downloadPromise = writer.closed.catch(() => {throw DOWNLOAD_ERROR;}); + const item: DownloadItem = { + ...payload, + // transformStream, + readableStream: readable, + // writableStream: writable, + // writer, + // downloadPromise, + promise, + controller + }; + + downloadMap.set(id, item); + + // return downloadPromise; + return promise.catch(() => {throw DOWNLOAD_ERROR}); + }, + + downloadChunk: ({id, chunk}) => { + const item = downloadMap.get(id); + if(!item) { + return Promise.reject(); + } + + // return item.controller.enqueue(chunk); + // return item.writer.write(chunk); + return item.controller.enqueue(chunk); + }, + + downloadFinalize: (id) => { + const item = downloadMap.get(id); + if(!item) { + return Promise.reject(); + } + + item.promise.resolve(); + // return item.controller.terminate(); + // return item.writer.close(); + return item.controller.close(); + }, + + downloadCancel: (id) => { + const item = downloadMap.get(id); + if(!item) { + return; + } + + item.promise.reject(); + // return item.controller.error(); + // return item.writer.abort(); + return item.controller.error(); + } +}; + +export default function handleDownload(serviceMessagePort: ServiceMessagePort) { + serviceMessagePort.addMultipleEventsListeners(events); + + return { + onDownloadFetch, + onClosedWindows: cancelAllDownloads + }; +} + +function onDownloadFetch(event: FetchEvent, params: string) { + event.respondWith(pause(100).then(() => { + const item = downloadMap.get(params); + if(!item || (item.used && !DOWNLOAD_TEST)) { + return; + } + + item.used = true; + const stream = item.readableStream; + const response = new Response(stream, {headers: item.headers}); + return response; + })); + + // event.respondWith(response); +} + +function cancelAllDownloads() { + if(downloadMap.size) { + for(const [id, item] of downloadMap) { + // item.writer.abort().catch(noop); + item.controller.error(); + } + } +} diff --git a/src/lib/serviceWorker/index.service.ts b/src/lib/serviceWorker/index.service.ts index 1d8b0a9f..3602b636 100644 --- a/src/lib/serviceWorker/index.service.ts +++ b/src/lib/serviceWorker/index.service.ts @@ -14,12 +14,11 @@ import onStreamFetch from './stream'; import {closeAllNotifications, onPing} from './push'; import CacheStorageController from '../files/cacheStorage'; import {IS_SAFARI} from '../../environment/userAgent'; -import ServiceMessagePort, {ServiceDownloadTaskPayload} from './serviceMessagePort'; +import ServiceMessagePort from './serviceMessagePort'; import listenMessagePort from '../../helpers/listenMessagePort'; import {getWindowClients} from '../../helpers/context'; import {MessageSendPort} from '../mtproto/superMessagePort'; -import noop from '../../helpers/noop'; -import makeError from '../../helpers/makeError'; +import handleDownload from './download'; export const log = logger('SW', LogTypes.Error | LogTypes.Debug | LogTypes.Log | LogTypes.Warn); const ctx = self as any as ServiceWorkerGlobalScope; @@ -52,19 +51,6 @@ const onWindowConnected = (source: WindowClient) => { connectedWindows.add(source.id); }; -type DownloadType = Uint8Array; -type DownloadItem = ServiceDownloadTaskPayload & { - transformStream: TransformStream, - readableStream: ReadableStream, - writableStream: WritableStream, - writer: WritableStreamDefaultWriter, - // controller: TransformStreamDefaultController, - // promise: CancellablePromise, - used?: boolean -}; -const downloadMap: Map = new Map(); -const DOWNLOAD_ERROR = makeError('UNKNOWN'); - export const serviceMessagePort = new ServiceMessagePort(); serviceMessagePort.addMultipleEventsListeners({ notificationsClear: closeAllNotifications, @@ -79,86 +65,14 @@ serviceMessagePort.addMultipleEventsListeners({ hello: (payload, source) => { onWindowConnected(source as any as WindowClient); - }, - - download: (payload) => { - const {id} = payload; - if(downloadMap.has(id)) { - return; - } - - // const writableStrategy = new ByteLengthQueuingStrategy({highWaterMark: 1024 * 1024}); - // let controller: TransformStreamDefaultController; - const transformStream = new TransformStream(/* { - start: (_controller) => controller = _controller, - }, { - highWaterMark: 1, - size: (chunk) => chunk.byteLength - }, new CountQueuingStrategy({highWaterMark: 4}) */); - - const {readable, writable} = transformStream; - const writer = writable.getWriter(); - // const promise = deferredPromise(); - // promise.catch(noop).finally(() => { - // downloadMap.delete(id); - // }); - - // writer.closed.then(promise.resolve, promise.reject); - - writer.closed.catch(noop).finally(() => { - log.error('closed writer'); - downloadMap.delete(id); - }); - - const item: DownloadItem = { - ...payload, - transformStream, - readableStream: readable, - writableStream: writable, - writer - // promise, - // controller - }; - - downloadMap.set(id, item); - - return writer.closed.catch(() => {throw DOWNLOAD_ERROR;}); - // return promise; - }, - - downloadChunk: ({id, chunk}) => { - const item = downloadMap.get(id); - if(!item) { - return Promise.reject(); - } - - // return item.controller.enqueue(chunk); - return item.writer.write(chunk); - }, - - downloadFinalize: (id) => { - const item = downloadMap.get(id); - if(!item) { - return Promise.reject(); - } - - // item.promise.resolve(); - // return item.controller.terminate(); - return item.writer.close(); - }, - - downloadCancel: (id) => { - const item = downloadMap.get(id); - if(!item) { - return; - } - - // item.promise.reject(); - // return item.controller.error(); - return item.writer.abort(); } }); +const { + onDownloadFetch, + onClosedWindows: onDownloadClosedWindows +} = handleDownload(serviceMessagePort); + // * service worker can be killed, so won't get 'hello' event getWindowClients().then((windowClients) => { log(`got ${windowClients.length} windows from the start`); @@ -184,11 +98,7 @@ listenMessagePort(serviceMessagePort, undefined, (source) => { _mtprotoMessagePort = undefined; } - if(downloadMap.size) { - for(const [id, item] of downloadMap) { - item.writer.abort().catch(noop); - } - } + onDownloadClosedWindows(); } }); // #endif @@ -216,14 +126,7 @@ const onFetch = (event: FetchEvent): void => { } case 'download': { - const item = downloadMap.get(params); - if(!item || item.used) { - break; - } - - item.used = true; - const response = new Response(item.transformStream.readable, {headers: item.headers}); - event.respondWith(response); + onDownloadFetch(event, params); break; } } @@ -231,7 +134,8 @@ const onFetch = (event: FetchEvent): void => { log.error('fetch error', err); event.respondWith(new Response('', { status: 500, - statusText: 'Internal Server Error' + statusText: 'Internal Server Error', + headers: {'Cache-Control': 'no-cache'} })); } }; @@ -242,13 +146,13 @@ const onChangeState = () => { ctx.addEventListener('install', (event) => { log('installing'); - event.waitUntil(ctx.skipWaiting()); // Activate worker immediately + event.waitUntil(ctx.skipWaiting().then(() => log('skipped waiting'))); // Activate worker immediately }); ctx.addEventListener('activate', (event) => { log('activating', ctx); - event.waitUntil(ctx.caches.delete(CACHE_ASSETS_NAME)); - event.waitUntil(ctx.clients.claim()); + event.waitUntil(ctx.caches.delete(CACHE_ASSETS_NAME).then(() => log('cleared assets cache'))); + event.waitUntil(ctx.clients.claim().then(() => log('claimed clients'))); }); // ctx.onerror = (error) => { diff --git a/src/lib/serviceWorker/serviceMessagePort.ts b/src/lib/serviceWorker/serviceMessagePort.ts index 290ea2c5..c4a2975c 100644 --- a/src/lib/serviceWorker/serviceMessagePort.ts +++ b/src/lib/serviceWorker/serviceMessagePort.ts @@ -28,9 +28,8 @@ export type ServiceRequestFilePartTaskPayload = { }; export type ServiceDownloadTaskPayload = { - fileName: string, headers: any, - id: string, + id: string }; export type ServiceEvent = { diff --git a/src/scss/partials/_document.scss b/src/scss/partials/_document.scss index 4f0d9aee..870f4750 100644 --- a/src/scss/partials/_document.scss +++ b/src/scss/partials/_document.scss @@ -23,6 +23,10 @@ line-height: 1; text-align: center; + html.is-safari & { + -webkit-mask-image: -webkit-radial-gradient(circle, white 100%, black 100%); // fix safari overflow + } + &-text { opacity: 0;