Browse Source

Better downloading

master
Eduard Kuzmenko 2 years ago
parent
commit
439d2ce684
  1. 6
      .eslintrc.js
  2. 1
      src/components/chat/bubbles.ts
  3. 18
      src/components/wrappers/document.ts
  4. 2
      src/global.d.ts
  5. 2
      src/helpers/fileName.ts
  6. 14
      src/helpers/formatBytes.ts
  7. 9
      src/lib/appManagers/appDocsManager.ts
  8. 132
      src/lib/appManagers/appDownloadManager.ts
  9. 2
      src/lib/appManagers/utils/download/getDownloadFileNameFromOptions.ts
  10. 2
      src/lib/appManagers/utils/webDocs/getWebDocumentDownloadOptions.ts
  11. 16
      src/lib/files/cacheStorage.ts
  12. 55
      src/lib/files/downloadStorage.ts
  13. 29
      src/lib/files/downloadWriter.ts
  14. 4
      src/lib/files/fileStorage.ts
  15. 533
      src/lib/mtproto/apiFileManager.ts
  16. 72
      src/lib/mtproto/api_methods.ts
  17. 2
      src/lib/mtproto/dcConfigurator.ts
  18. 4
      src/lib/mtproto/transports/websocket.ts
  19. 3
      src/lib/rootScope.ts
  20. 162
      src/lib/serviceWorker/download.ts
  21. 124
      src/lib/serviceWorker/index.service.ts
  22. 3
      src/lib/serviceWorker/serviceMessagePort.ts
  23. 4
      src/scss/partials/_document.scss

6
.eslintrc.js

@ -6,9 +6,9 @@ module.exports = { @@ -6,9 +6,9 @@ module.exports = {
extends: [],
parser: '@typescript-eslint/parser',
parserOptions: {
// 'ecmaVersion': 'latest',
// 'sourceType': 'module'
project: ['./tsconfig.json']
'ecmaVersion': 'latest',
'sourceType': 'module'
// project: ['./tsconfig.json']
},
plugins: [
'@typescript-eslint'

1
src/components/chat/bubbles.ts

@ -522,6 +522,7 @@ export default class ChatBubbles { @@ -522,6 +522,7 @@ export default class ChatBubbles {
(element as any).onLoad(true);
} else {
element.dataset.docId = '' + doc.id;
(element as any).doc = doc;
}
}
} else if(poll) {

18
src/components/wrappers/document.ts

@ -90,6 +90,7 @@ export default async function wrapDocument({message, withTime, fontWeight, voice @@ -90,6 +90,7 @@ export default async function wrapDocument({message, withTime, fontWeight, voice
const docDiv = document.createElement('div');
docDiv.classList.add('document', `ext-${ext}`);
docDiv.dataset.docId = '' + doc.id;
(docDiv as any).doc = doc;
// return docDiv;
@ -236,25 +237,28 @@ export default async function wrapDocument({message, withTime, fontWeight, voice @@ -236,25 +237,28 @@ export default async function wrapDocument({message, withTime, fontWeight, voice
// b && b.classList.add('hide');
let d = formatBytes(0);
const format = (bytes: number) => formatBytes(bytes);
let d = format(0);
bytesContainer.style.visibility = 'hidden';
// bytesContainer.replaceWith(sizeContainer);
sizeContainer.append(d, bytesJoiner, _bytesContainer);
bytesContainer.parentElement.append(sizeContainer);
promise.addNotifyListener((progress: Progress) => {
const _d = formatBytes(progress.done);
const _d = format(progress.done);
d.replaceWith(_d);
d = _d;
});
};
const load = async(e?: Event) => {
// ! DO NOT USE ASYNC/AWAIT HERE ! SAFARI WON'T LET DOWNLOAD THE FILE BECAUSE OF ASYNC
const load = (e?: Event) => {
const save = !e || e.isTrusted;
const doc = await managers.appDocsManager.getDoc(docDiv.dataset.docId);
const doc = (docDiv as any).doc;
// const doc = await managers.appDocsManager.getDoc(docDiv.dataset.docId);
let download: CancellablePromise<any>;
const queueId = appImManager.chat.bubbles ? appImManager.chat.bubbles.lazyLoadQueue.queueId : undefined;
if(!save) {
download = appDownloadManager.downloadMediaVoid({media: doc, queueId});
download = appDownloadManager.downloadToDisc({media: doc, queueId}, true);
} else if(doc.type === 'pdf') {
const canOpenAfter = /* managers.appDocsManager.downloading.has(doc.id) || */!preloader || preloader.detached;
download = appDownloadManager.downloadMediaURL({media: doc, queueId});
@ -282,10 +286,10 @@ export default async function wrapDocument({message, withTime, fontWeight, voice @@ -282,10 +286,10 @@ export default async function wrapDocument({message, withTime, fontWeight, voice
}
};
const {fileName: downloadFileName} = getDownloadMediaDetails({media: doc});
const {fileName: downloadFileName} = getDownloadMediaDetails({media: doc, downloadId: '1'});
if(await managers.apiFileManager.isDownloading(downloadFileName)) {
downloadDiv = docDiv.querySelector('.document-download') || icoDiv;
const promise = appDownloadManager.downloadMediaVoid({media: doc});
const promise = appDownloadManager.downloadToDisc({media: doc}, true);
preloader = new ProgressivePreloader();
preloader.attach(downloadDiv, false, promise);

2
src/global.d.ts vendored

@ -30,7 +30,7 @@ declare global { @@ -30,7 +30,7 @@ declare global {
type FiltersError = 'PINNED_DIALOGS_TOO_MUCH';
type LocalFileError = ApiFileManagerError | ReferenceError | StorageError;
type LocalErrorType = LocalFileError | NetworkerError | FiltersError | 'UNKNOWN';
type LocalErrorType = LocalFileError | NetworkerError | FiltersError | 'UNKNOWN' | 'NO_DOC';
type ServerErrorType = 'FILE_REFERENCE_EXPIRED' | 'SESSION_REVOKED' | 'AUTH_KEY_DUPLICATED' |
'SESSION_PASSWORD_NEEDED' | 'CONNECTION_NOT_INITED' | 'ERROR_EMPTY' | 'MTPROTO_CLUSTER_INVALID' |

2
src/helpers/fileName.ts

@ -58,7 +58,7 @@ export function getFileNameByLocation(location: InputFileLocation | InputWebFile @@ -58,7 +58,7 @@ export function getFileNameByLocation(location: InputFileLocation | InputWebFile
}
}
return str + (options.downloadId || '') + (ext ? '.' + ext : ext);
return str + (options.downloadId ? '_download' : '') + (ext ? '.' + ext : ext);
}
export type FileURLType = 'photo' | 'thumb' | 'document' | 'stream' | 'download';

14
src/helpers/formatBytes.ts

@ -6,14 +6,18 @@ @@ -6,14 +6,18 @@
import {i18n, LangPackKey} from '../lib/langPack';
export default function formatBytes(bytes: number, decimals = 2) {
export default function formatBytes(bytes: number, decimals: number | 'auto' = 'auto') {
if(bytes === 0) return i18n('FileSize.B', [0]);
const k = 1024;
const dm = decimals < 0 ? 0 : decimals;
const sizes: LangPackKey[] = ['FileSize.B', 'FileSize.KB', 'FileSize.MB', 'FileSize.GB'];
const strictDecimals = decimals === 'auto';
const k = 1024;
const i = Math.floor(Math.log(bytes) / Math.log(k));
const _decimals = decimals === 'auto' ? Math.max(0, i - 1) : decimals;
const dm = Math.max(0, _decimals);
const sizes: LangPackKey[] = ['FileSize.B', 'FileSize.KB', 'FileSize.MB', 'FileSize.GB'];
return i18n(sizes[i], [parseFloat((bytes / Math.pow(k, i)).toFixed(dm))]);
const fixed = (bytes / Math.pow(k, i)).toFixed(dm);
return i18n(sizes[i], [strictDecimals ? fixed : parseFloat(fixed)]);
}

9
src/lib/appManagers/appDocsManager.ts

@ -9,7 +9,7 @@ @@ -9,7 +9,7 @@
* https://github.com/zhukov/webogram/blob/master/LICENSE
*/
import {AccountWallPapers, Document, MessagesSavedGifs, PhotoSize, WallPaper} from '../../layer';
import {AccountWallPapers, Document, DocumentAttribute, MessagesSavedGifs, PhotoSize, WallPaper} from '../../layer';
import {ReferenceContext} from '../mtproto/referenceDatabase';
import {getFullDate} from '../../helpers/date';
import isObject from '../../helpers/object/isObject';
@ -23,6 +23,7 @@ import MTProtoMessagePort from '../mtproto/mtprotoMessagePort'; @@ -23,6 +23,7 @@ import MTProtoMessagePort from '../mtproto/mtprotoMessagePort';
import getDocumentInput from './utils/docs/getDocumentInput';
import getDocumentURL from './utils/docs/getDocumentURL';
import type {ThumbCache} from '../storages/thumbs';
import makeError from '../../helpers/makeError';
export type MyDocument = Document.document;
@ -216,7 +217,10 @@ export class AppDocsManager extends AppManager { @@ -216,7 +217,10 @@ export class AppDocsManager extends AppManager {
if(doc.type === 'voice' || doc.type === 'round') {
// browser will identify extension
doc.file_name = doc.type + '_' + getFullDate(new Date(doc.date * 1000), {monthAsNumber: true, leadingZero: true}).replace(/[:\.]/g, '-').replace(', ', '_');
const attribute = doc.attributes.find((attribute) => attribute._ === 'documentAttributeFilename') as DocumentAttribute.documentAttributeFilename;
const ext = attribute && attribute.file_name.split('.').pop();
const date = getFullDate(new Date(doc.date * 1000), {monthAsNumber: true, leadingZero: true}).replace(/[:\.]/g, '-').replace(', ', '_');
doc.file_name = `${doc.type}_${date}${ext ? '.' + ext : ''}`;
}
if(isServiceWorkerOnline()) {
@ -400,6 +404,7 @@ export class AppDocsManager extends AppManager { @@ -400,6 +404,7 @@ export class AppDocsManager extends AppManager {
public requestDocPart(docId: DocId, dcId: number, offset: number, limit: number) {
const doc = this.getDoc(docId);
if(!doc) return Promise.reject(makeError('NO_DOC'));
return this.apiFileManager.requestFilePart(dcId, getDocumentInput(doc), offset, limit);
}
}

132
src/lib/appManagers/appDownloadManager.ts

@ -16,8 +16,7 @@ import noop from '../../helpers/noop'; @@ -16,8 +16,7 @@ import noop from '../../helpers/noop';
import getDownloadMediaDetails from './utils/download/getDownloadMediaDetails';
import getDownloadFileNameFromOptions from './utils/download/getDownloadFileNameFromOptions';
import indexOfAndSplice from '../../helpers/array/indexOfAndSplice';
import {MAX_FILE_SAVE_SIZE} from '../mtproto/mtproto_config';
import createDownloadAnchor from '../../helpers/dom/createDownloadAnchor';
import makeError from '../../helpers/makeError';
export type ResponseMethodBlob = 'blob';
export type ResponseMethodJson = 'json';
@ -38,6 +37,7 @@ type DownloadType = 'url' | 'blob' | 'void' | 'disc'; @@ -38,6 +37,7 @@ type DownloadType = 'url' | 'blob' | 'void' | 'disc';
export class AppDownloadManager {
private downloads: {[fileName: string]: {main: Download} & {[type in DownloadType]?: Download}} = {};
// private downloadsToDisc: {[fileName: string]: Download} = {};
private progress: {[fileName: string]: Progress} = {};
// private progressCallbacks: {[fileName: string]: Array<ProgressCallback>} = {};
private managers: AppManagers;
@ -45,15 +45,14 @@ export class AppDownloadManager { @@ -45,15 +45,14 @@ export class AppDownloadManager {
public construct(managers: AppManagers) {
this.managers = managers;
rootScope.addEventListener('download_progress', (details) => {
this.progress[details.fileName] = details;
// const callbacks = this.progressCallbacks[details.fileName];
// if(callbacks) {
// callbacks.forEach((callback) => callback(details));
// }
const download = this.downloads[details.fileName];
if(download) {
if(download?.main?.notifyAll) {
this.progress[details.fileName] = details;
download.main.notifyAll(details);
}
});
@ -69,17 +68,12 @@ export class AppDownloadManager { @@ -69,17 +68,12 @@ export class AppDownloadManager {
};
deferred.cancel = () => {
// try {
const error = new Error('Download canceled');
error.name = 'AbortError';
const error = makeError('DOWNLOAD_CANCELED');
this.managers.apiFileManager.cancelDownload(fileName);
deferred.reject(error);
deferred.cancel = () => {};
/* } catch(err) {
} */
deferred.cancel = noop;
};
deferred.catch(() => {
@ -225,51 +219,93 @@ export class AppDownloadManager { @@ -225,51 +219,93 @@ export class AppDownloadManager {
// }
// }
public downloadToDisc(options: DownloadMediaOptions) {
public downloadToDisc(options: DownloadMediaOptions, justAttach?: boolean) {
const media = options.media;
const isDocument = media._ === 'document';
if(!isDocument && !options.thumb) {
options.thumb = (media as Photo.photo).sizes.slice().pop() as PhotoSize.photoSize;
}
const {downloadOptions, fileName} = getDownloadMediaDetails(options);
if(downloadOptions.size && downloadOptions.size > MAX_FILE_SAVE_SIZE) {
const id = '' + (Math.random() * 0x7FFFFFFF | 0);
const url = `/download/${id}`;
options.downloadId = id;
const promise = this.downloadMedia(options, 'disc');
let iframe: HTMLIFrameElement;
const onProgress = () => {
iframe = document.createElement('iframe');
iframe.hidden = true;
// iframe.src = sw.scope + fileName;
iframe.src = url;
document.body.append(iframe);
indexOfAndSplice(promise.listeners, onProgress);
};
promise.addNotifyListener(onProgress);
promise.catch(noop).finally(() => {
setTimeout(() => {
iframe?.remove();
}, 1000);
});
return promise;
} else {
const promise = this.downloadMedia(options, 'blob');
promise.then((blob) => {
const url = URL.createObjectURL(blob);
createDownloadAnchor(url, downloadOptions.fileName || fileName, () => {
URL.revokeObjectURL(url);
});
});
// const {fileName: cacheFileName} = getDownloadMediaDetails(options);
// if(justAttach) {
// const promise = this.downloadsToDisc[cacheFileName];
// if(promise) {
// return promise;
// }
// }
// const {downloadOptions, fileName} = getDownloadMediaDetails(options);
// if(downloadOptions.size && downloadOptions.size > MAX_FILE_SAVE_SIZE) {
const id = '' + (Math.random() * 0x7FFFFFFF | 0);
// const id = 'test';
const url = `/download/${id}`;
options.downloadId = id;
const promise = this.downloadMedia(options, 'disc');
// this.downloadsToDisc[cacheFileName] = promise;
if(justAttach) {
return promise;
}
const iframe = document.createElement('iframe');
iframe.hidden = true;
iframe.src = url;
document.body.append(iframe);
// createDownloadAnchor(url, 'asd.txt');
// const events = [
// 'emptied',
// 'abort',
// 'suspend',
// 'reset',
// 'error',
// 'ended',
// 'load'
// ].forEach((event) => {
// iframe.addEventListener(event, () => alert(event));
// iframe.contentWindow.addEventListener(event, () => alert(event));
// });
let element: HTMLElement, hadProgress = false;
const onProgress = () => {
if(hadProgress) {
return;
}
hadProgress = true;
element = iframe;
indexOfAndSplice(promise.listeners, onProgress);
};
promise.addNotifyListener(onProgress);
promise.catch(noop).finally(() => {
if(!hadProgress) {
onProgress();
}
setTimeout(() => {
element?.remove();
}, 1000);
// if(this.downloadsToDisc[cacheFileName] === promise) {
// delete this.downloadsToDisc[cacheFileName];
// }
});
return promise;
// } else {
// const promise = this.downloadMedia(options, 'blob');
// promise.then((blob) => {
// const url = URL.createObjectURL(blob);
// createDownloadAnchor(url, downloadOptions.fileName || fileName, () => {
// URL.revokeObjectURL(url);
// });
// });
// return promise;
// }
// const promise = this.downloadMedia(options);
// promise.then((blob) => {
// const url = URL.createObjectURL(blob);

2
src/lib/appManagers/utils/download/getDownloadFileNameFromOptions.ts

@ -8,5 +8,5 @@ import {getFileNameByLocation} from '../../../../helpers/fileName'; @@ -8,5 +8,5 @@ import {getFileNameByLocation} from '../../../../helpers/fileName';
import {DownloadOptions} from '../../../mtproto/apiFileManager';
export default function getDownloadFileNameFromOptions(options: DownloadOptions) {
return getFileNameByLocation(options.location, {fileName: options.fileName});
return getFileNameByLocation(options.location, options);
}

2
src/lib/appManagers/utils/webDocs/getWebDocumentDownloadOptions.ts

@ -3,7 +3,7 @@ import {DownloadOptions} from '../../../mtproto/apiFileManager'; @@ -3,7 +3,7 @@ import {DownloadOptions} from '../../../mtproto/apiFileManager';
export default function getWebDocumentDownloadOptions(webDocument: WebDocument): DownloadOptions {
return {
dcId: 4,
dcId: 0,
location: {
_: 'inputWebFileLocation',
access_hash: (webDocument as WebDocument.webDocument).access_hash,

16
src/lib/files/cacheStorage.ts

@ -10,6 +10,7 @@ import MemoryWriter from './memoryWriter'; @@ -10,6 +10,7 @@ import MemoryWriter from './memoryWriter';
import FileManager from './memoryWriter';
import FileStorage from './fileStorage';
import makeError from '../../helpers/makeError';
import deferredPromise from '../../helpers/cancellablePromise';
export type CacheStorageDbName = 'cachedFiles' | 'cachedStreamChunks' | 'cachedAssets';
@ -124,12 +125,17 @@ export default class CacheStorageController implements FileStorage { @@ -124,12 +125,17 @@ export default class CacheStorageController implements FileStorage {
});
}
public getWriter(fileName: string, fileSize: number, mimeType: string) {
const writer = new MemoryWriter(mimeType, fileSize, (blob) => {
return this.saveFile(fileName, blob).catch(() => blob);
});
public prepareWriting(fileName: string, fileSize: number, mimeType: string) {
return {
deferred: deferredPromise<Blob>(),
getWriter: () => {
const writer = new MemoryWriter(mimeType, fileSize, (blob) => {
return this.saveFile(fileName, blob).catch(() => blob);
});
return Promise.resolve(writer);
return writer;
}
};
}
public static toggleStorage(enabled: boolean, clearWrite: boolean) {

55
src/lib/files/downloadStorage.ts

@ -0,0 +1,55 @@ @@ -0,0 +1,55 @@
/*
* https://github.com/morethanwords/tweb
* Copyright (C) 2019-2021 Eduard Kuzmenko
* https://github.com/morethanwords/tweb/blob/master/LICENSE
*/
import deferredPromise from '../../helpers/cancellablePromise';
import makeError from '../../helpers/makeError';
import fileNameRFC from '../../helpers/string/fileNameRFC';
import {getServiceMessagePort} from '../mtproto/mtproto.worker';
import DownloadWriter from './downloadWriter';
import FileStorage from './fileStorage';
import StreamWriter from './streamWriter';
export default class DownloadStorage implements FileStorage {
public getFile(fileName: string): Promise<any> {
return Promise.reject(makeError('NO_ENTRY_FOUND'));
}
public prepareWriting({fileName, downloadId, size}: {
fileName: string,
downloadId: string,
size: number
}) {
const headers = {
'Content-Type': 'application/octet-stream; charset=utf-8',
'Content-Disposition': 'attachment; filename*=UTF-8\'\'' + fileNameRFC(fileName),
...(size ? {'Content-Length': size} : {})
};
const serviceMessagePort = getServiceMessagePort();
const promise = serviceMessagePort.invoke('download', {
headers,
id: downloadId
});
const deferred = deferredPromise<void>();
deferred.cancel = () => {
deferred.reject(makeError('DOWNLOAD_CANCELED'));
};
deferred.catch(() => {
getServiceMessagePort().invoke('downloadCancel', downloadId);
});
promise.then(deferred.resolve, deferred.reject);
return {
deferred,
getWriter: () => {
return new DownloadWriter(serviceMessagePort, downloadId);
}
};
}
}

29
src/lib/files/downloadWriter.ts

@ -0,0 +1,29 @@ @@ -0,0 +1,29 @@
/*
* https://github.com/morethanwords/tweb
* Copyright (C) 2019-2021 Eduard Kuzmenko
* https://github.com/morethanwords/tweb/blob/master/LICENSE
*/
import {getServiceMessagePort} from '../mtproto/mtproto.worker';
import ServiceMessagePort from '../serviceWorker/serviceMessagePort';
import StreamWriter from './streamWriter';
export default class DownloadWriter implements StreamWriter {
constructor(
private serviceMessagePort: ServiceMessagePort<true>,
private downloadId: string
) {
this.serviceMessagePort = getServiceMessagePort();
}
public async write(part: Uint8Array, offset?: number) {
return this.serviceMessagePort.invoke('downloadChunk', {
id: this.downloadId,
chunk: part
});
}
public finalize(saveToStorage?: boolean): Promise<Blob> {
return this.serviceMessagePort.invoke('downloadFinalize', this.downloadId).then(() => undefined);
}
}

4
src/lib/files/fileStorage.ts

@ -4,10 +4,10 @@ @@ -4,10 +4,10 @@
* https://github.com/morethanwords/tweb/blob/master/LICENSE
*/
import {CancellablePromise} from '../../helpers/cancellablePromise';
import StreamWriter from './streamWriter';
export default abstract class FileStorage {
public abstract getFile(fileName: string): Promise<any>;
public abstract getWriter(fileName: string, fileSize: number, mimeType: string): Promise<StreamWriter>;
public abstract prepareWriting(...args: any[]): {deferred: CancellablePromise<any>, getWriter: () => StreamWriter};
}

533
src/lib/mtproto/apiFileManager.ts

@ -18,7 +18,6 @@ import {DcId} from '../../types'; @@ -18,7 +18,6 @@ import {DcId} from '../../types';
import CacheStorageController from '../files/cacheStorage';
import {logger, LogTypes} from '../logger';
import assumeType from '../../helpers/assumeType';
import ctx from '../../environment/ctx';
import noop from '../../helpers/noop';
import readBlobAsArrayBuffer from '../../helpers/blob/readBlobAsArrayBuffer';
import bytesToHex from '../../helpers/bytes/bytesToHex';
@ -32,13 +31,15 @@ import type {Progress} from '../appManagers/appDownloadManager'; @@ -32,13 +31,15 @@ import type {Progress} from '../appManagers/appDownloadManager';
import getDownloadMediaDetails from '../appManagers/utils/download/getDownloadMediaDetails';
import networkStats from './networkStats';
import getDownloadFileNameFromOptions from '../appManagers/utils/download/getDownloadFileNameFromOptions';
import {getServiceMessagePort} from './mtproto.worker';
import StreamWriter from '../files/streamWriter';
import FileStorage from '../files/fileStorage';
import fileNameRFC from '../../helpers/string/fileNameRFC';
import {MAX_FILE_SAVE_SIZE} from './mtproto_config';
import throttle from '../../helpers/schedulers/throttle';
import makeError from '../../helpers/makeError';
import readBlobAsUint8Array from '../../helpers/blob/readBlobAsUint8Array';
import DownloadStorage from '../files/downloadStorage';
import copy from '../../helpers/object/copy';
import indexOfAndSplice from '../../helpers/array/indexOfAndSplice';
type Delayed = {
offset: number,
@ -82,10 +83,13 @@ export type MyUploadFile = UploadFile.uploadFile | UploadWebFile.uploadWebFile; @@ -82,10 +83,13 @@ export type MyUploadFile = UploadFile.uploadFile | UploadWebFile.uploadWebFile;
// originalPayload: ReferenceBytes
// };
const MAX_FILE_PART_SIZE = 1 * 1024 * 1024;
const MAX_DOWNLOAD_FILE_PART_SIZE = 1 * 1024 * 1024;
const MAX_UPLOAD_FILE_PART_SIZE = 512 * 1024;
const MIN_PART_SIZE = 128 * 1024;
const AVG_PART_SIZE = 512 * 1024;
const REGULAR_DOWNLOAD_DELTA = 36;
const PREMIUM_DOWNLOAD_DELTA = 72;
const REGULAR_DOWNLOAD_DELTA = (9 * 512 * 1024) / MIN_PART_SIZE;
const PREMIUM_DOWNLOAD_DELTA = REGULAR_DOWNLOAD_DELTA * 2;
const IGNORE_ERRORS: Set<ErrorType> = new Set([
'DOWNLOAD_CANCELED',
@ -96,11 +100,16 @@ const IGNORE_ERRORS: Set<ErrorType> = new Set([ @@ -96,11 +100,16 @@ const IGNORE_ERRORS: Set<ErrorType> = new Set([
export class ApiFileManager extends AppManager {
private cacheStorage = new CacheStorageController('cachedFiles');
private downloadStorage = new DownloadStorage();
private downloadPromises: {
[fileName: string]: DownloadPromise
} = {};
// private downloadToDiscPromises: {
// [fileName: string]: DownloadPromise
// } = {};
private uploadPromises: {
[fileName: string]: CancellablePromise<InputFile>
} = {};
@ -133,6 +142,7 @@ export class ApiFileManager extends AppManager { @@ -133,6 +142,7 @@ export class ApiFileManager extends AppManager {
private maxUploadParts = 4000;
private maxDownloadParts = 8000;
private webFileDcId: DcId;
protected after() {
setInterval(() => { // clear old promises
@ -144,6 +154,10 @@ export class ApiFileManager extends AppManager { @@ -144,6 +154,10 @@ export class ApiFileManager extends AppManager {
}
}, 1800e3);
this.rootScope.addEventListener('config', (config) => {
this.webFileDcId = config.webfile_dc_id;
});
this.rootScope.addEventListener('app_config', (appConfig) => {
this.maxUploadParts = this.rootScope.premium ? appConfig.upload_max_fileparts_premium : appConfig.upload_max_fileparts_default;
this.maxDownloadParts = appConfig.upload_max_fileparts_premium;
@ -173,7 +187,7 @@ export class ApiFileManager extends AppManager { @@ -173,7 +187,7 @@ export class ApiFileManager extends AppManager {
private downloadCheck(dcId: string | number) {
const downloadPull = this.downloadPulls[dcId];
const downloadLimit = dcId === 'upload' ? 24 : (this.rootScope.premium ? PREMIUM_DOWNLOAD_DELTA : REGULAR_DOWNLOAD_DELTA);
const downloadLimit = /* dcId === 'upload' ? 24 : */(this.rootScope.premium ? PREMIUM_DOWNLOAD_DELTA : REGULAR_DOWNLOAD_DELTA);
// const downloadLimit = Infinity;
if(this.downloadActives[dcId] >= downloadLimit || !downloadPull?.length) {
@ -187,7 +201,7 @@ export class ApiFileManager extends AppManager { @@ -187,7 +201,7 @@ export class ApiFileManager extends AppManager {
this.downloadActives[dcId] += activeDelta;
const promise = data.cb();
const networkPromise = networkStats.waitForChunk(dcId as DcId, activeDelta * 1024 * 128);
const networkPromise = networkStats.waitForChunk(dcId as DcId, activeDelta * MIN_PART_SIZE);
Promise.race([
promise,
networkPromise
@ -235,7 +249,7 @@ export class ApiFileManager extends AppManager { @@ -235,7 +249,7 @@ export class ApiFileManager extends AppManager {
public requestWebFilePart(dcId: DcId, location: InputWebFileLocation, offset: number, limit: number, id = 0, queueId = 0, checkCancel?: () => void) {
return this.downloadRequest(dcId, id, async() => { // do not remove async, because checkCancel will throw an error
checkCancel && checkCancel();
checkCancel?.();
return this.apiManager.invokeApi('upload.getWebFile', {
location,
@ -248,17 +262,26 @@ export class ApiFileManager extends AppManager { @@ -248,17 +262,26 @@ export class ApiFileManager extends AppManager {
}, this.getDelta(limit), queueId);
}
public requestFilePart(dcId: DcId, location: InputFileLocation, offset: number, limit: number, id = 0, queueId = 0, checkCancel?: () => void) {
public requestFilePart(
dcId: DcId,
location: InputFileLocation,
offset: number,
limit: number,
id = 0,
queueId = 0,
checkCancel?: () => void
) {
return this.downloadRequest(dcId, id, async() => { // do not remove async, because checkCancel will throw an error
checkCancel && checkCancel();
checkCancel?.();
const invoke = async(): Promise<MyUploadFile> => {
checkCancel && checkCancel(); // do not remove async, because checkCancel will throw an error
checkCancel?.(); // do not remove async, because checkCancel will throw an error
// * IMPORTANT: reference can be changed in previous request
const reference = (location as InputFileLocation.inputDocumentFileLocation).file_reference?.slice();
const promise = /* pause(1000).then(() => */this.apiManager.invokeApi('upload.getFile', {
const promise = // pause(offset > (100 * 1024 * 1024) ? 10000000 : 0).then(() =>
this.apiManager.invokeApi('upload.getFile', {
location,
offset,
limit
@ -268,6 +291,8 @@ export class ApiFileManager extends AppManager { @@ -268,6 +291,8 @@ export class ApiFileManager extends AppManager {
}) as Promise<MyUploadFile>/* ) */;
return promise.catch((err: ApiError) => {
checkCancel?.();
if(err.type === 'FILE_REFERENCE_EXPIRED') {
return this.refreshReference(location as InputFileLocation.inputDocumentFileLocation, reference).then(invoke);
}
@ -295,19 +320,22 @@ export class ApiFileManager extends AppManager { @@ -295,19 +320,22 @@ export class ApiFileManager extends AppManager {
} */
private getDelta(bytes: number) {
return bytes / 1024 / 128;
return bytes / MIN_PART_SIZE;
}
private getLimitPart(size: number, isUpload: boolean): number {
if(!size) { // * sometimes size can be 0 (e.g. avatars, webDocuments)
return 512 * 1024;
return AVG_PART_SIZE;
}
let bytes = 128 * 1024;
// return 1 * 1024 * 1024;
let bytes = MIN_PART_SIZE;
const maxParts = isUpload ? this.maxUploadParts : this.maxDownloadParts;
const maxPartSize = isUpload ? MAX_UPLOAD_FILE_PART_SIZE : MAX_DOWNLOAD_FILE_PART_SIZE;
// usually it will stick to 512Kb size if the file is too big
while((size / bytes) > maxParts && bytes < MAX_FILE_PART_SIZE) {
while((size / bytes) > maxParts && bytes < maxPartSize) {
bytes *= 2;
}
/* if(size < 1e6 || !size) bytes = 512;
@ -399,201 +427,230 @@ export class ApiFileManager extends AppManager { @@ -399,201 +427,230 @@ export class ApiFileManager extends AppManager {
return this.uploadPromises[fileName];
}
public download(options: DownloadOptions): DownloadPromise {
const size = options.size ?? 0;
const {dcId, location, downloadId} = options;
private getConvertMethod(mimeType: string) {
let process: ApiFileManager['uncompressTGS'] | ApiFileManager['convertWebp'];
if(downloadId) {
} else if(options.mimeType === 'application/x-tgwallpattern') {
if(mimeType === 'application/x-tgwallpattern') {
process = this.uncompressTGV;
options.mimeType = 'image/svg+xml';
} else if(options.mimeType === 'image/webp' && !getEnvironment().IS_WEBP_SUPPORTED) {
mimeType = 'image/svg+xml';
} else if(mimeType === 'image/webp' && !getEnvironment().IS_WEBP_SUPPORTED) {
process = this.convertWebp;
options.mimeType = 'image/png';
} else if(options.mimeType === 'application/x-tgsticker') {
mimeType = 'image/png';
} else if(mimeType === 'application/x-tgsticker') {
process = this.uncompressTGS;
options.mimeType = 'application/json';
} else if(options.mimeType === 'audio/ogg' && !getEnvironment().IS_OPUS_SUPPORTED) {
mimeType = 'application/json';
} else if(mimeType === 'audio/ogg' && !getEnvironment().IS_OPUS_SUPPORTED) {
process = this.convertOpus;
options.mimeType = 'audio/wav';
mimeType = 'audio/wav';
}
const fileName = getDownloadFileNameFromOptions(options);
const cachedPromise = options.downloadId ? undefined : this.downloadPromises[fileName];
let fileStorage: FileStorage = this.getFileStorage();
return {mimeType, process};
}
this.debug && this.log('downloadFile', fileName, size, location, options.mimeType);
private allocateDeferredPromises(startOffset: number, size: number, limitPart: number) {
const delayed: Delayed[] = [];
let offset = startOffset;
let writePromise: CancellablePromise<void> = Promise.resolve(),
writeDeferred: CancellablePromise<void>;
do {
writeDeferred = deferredPromise<void>();
delayed.push({offset, writePromise, writeDeferred});
writePromise = writeDeferred;
offset += limitPart;
} while(offset < size);
return delayed;
}
/* if(options.queueId) {
this.log.error('downloadFile queueId:', fileName, options.queueId);
} */
public download(options: DownloadOptions): DownloadPromise {
const size = options.size ?? 0;
const {dcId, location, downloadId} = options;
if(cachedPromise) {
// this.log('downloadFile cachedPromise');
const originalMimeType = options.mimeType;
const convertMethod = this.getConvertMethod(originalMimeType);
const {process} = convertMethod;
options.mimeType = convertMethod.mimeType || 'image/jpeg';
if(size) {
return cachedPromise.then((blob) => {
if(blob instanceof Blob && blob.size < size) {
this.debug && this.log('downloadFile need to deleteFile, wrong size:', blob.size, size);
const fileName = getDownloadFileNameFromOptions(options);
const cacheFileName = downloadId ? getDownloadFileNameFromOptions({...copy(options), downloadId: undefined}) : fileName;
const cacheStorage: FileStorage = this.getFileStorage();
const downloadStorage: FileStorage = downloadId ? this.downloadStorage : undefined;
let deferred: DownloadPromise = downloadId ? undefined : this.downloadPromises[fileName];
return this.delete(fileName).then(() => {
return this.download(options);
}).catch(() => {
return this.download(options);
});
} else {
return blob;
}
});
} else {
return cachedPromise;
}
this.debug && this.log('downloadFile', fileName, options);
if(deferred) {
return deferred;
}
const deferred: DownloadPromise = deferredPromise();
const mimeType = options.mimeType || 'image/jpeg';
// if(deferred) {
// if(size) {
// return deferred.then(async(blob) => {
// if(blob instanceof Blob && blob.size < size) {
// this.debug && this.log('downloadFile need to deleteFile, wrong size:', blob.size, size);
// try {
// await this.delete(fileName);
// } finally {
// return this.download(options);
// }
// } else {
// return blob;
// }
// });
// } else {
// return deferred;
// }
// }
const errorHandler = (item: typeof cachePrepared, error: ApiError) => {
if(item?.error) {
return;
}
let error: ApiError;
let resolved = false;
let cacheWriter: StreamWriter;
let errorHandler = (_error: typeof error) => {
error = _error;
delete this.downloadPromises[fileName];
deferred.reject(error);
errorHandler = () => {};
for(const p of prepared) {
if(item && item !== p) {
continue;
}
if(cacheWriter && (!error || error.type !== 'DOWNLOAD_CANCELED')) {
cacheWriter.truncate?.();
p.error = error;
p.deferred.reject(error);
}
};
const id = this.tempId++;
const limitPart = options.limitPart || this.getLimitPart(size, false);
if(downloadId) {
const headers = {
'Content-Type': 'application/octet-stream; charset=utf-8',
'Content-Disposition': 'attachment; filename*=UTF-8\'\'' + fileNameRFC(options.fileName),
// 'Content-Disposition': `attachment; filename="${options.fileName}"`,
// 'Content-Type': 'application/octet-stream; charset=utf-8',
...(size ? {'Content-Length': size} : {})
};
let getFile: FileStorage['getFile'] = cacheStorage.getFile.bind(cacheStorage);
const serviceMessagePort = getServiceMessagePort();
const promise = serviceMessagePort.invoke('download', {
fileName,
headers,
id: downloadId
});
let cachePrepared: ReturnType<FileStorage['prepareWriting']> & {writer?: StreamWriter, error?: ApiError},
downloadPrepared: typeof cachePrepared;
const prepared: (typeof cachePrepared)[] = [];
const possibleSize = size || limitPart;
promise.catch(errorHandler);
deferred.catch(() => {
getServiceMessagePort().invoke('downloadCancel', downloadId);
});
const getErrorsCount = () => prepared.reduce((acc, item) => acc + +!!item.error, 0);
class f implements StreamWriter {
constructor() {
const attach = (item: typeof cachePrepared, fileName: string) => {
const {deferred} = item;
const _errorHandler = errorHandler.bind(null, item);
deferred.cancel = () => deferred.reject(makeError('DOWNLOAD_CANCELED'));
deferred.catch((error) => {
_errorHandler(error);
item.writer?.truncate?.();
}).finally(() => {
if(this.downloadPromises[fileName] === deferred) {
delete this.downloadPromises[fileName];
}
public async write(part: Uint8Array, offset?: number) {
return serviceMessagePort.invoke('downloadChunk', {
id: downloadId,
chunk: part
});
}
delete item.writer;
indexOfAndSplice(prepared, item);
});
public finalize(saveToStorage?: boolean): Promise<Blob> {
return serviceMessagePort.invoke('downloadFinalize', downloadId).then(() => null);
}
}
this.downloadPromises[fileName] = deferred;
class d implements FileStorage {
public getFile(fileName: string): Promise<any> {
return Promise.reject();
}
prepared.push(item);
};
public getWriter(fileName: string, fileSize: number, mimeType: string): Promise<StreamWriter> {
return Promise.resolve(new f());
}
if(cacheStorage && (!downloadStorage || possibleSize <= MAX_FILE_SAVE_SIZE)) {
cachePrepared = cacheStorage.prepareWriting(cacheFileName, possibleSize, options.mimeType)
attach(cachePrepared, cacheFileName);
}
if(downloadStorage) {
downloadPrepared = downloadStorage.prepareWriting({
fileName: options.fileName, // it's doc file_name
downloadId,
size: possibleSize
});
attach(downloadPrepared, fileName);
if(cachePrepared) { // cancel cache too
downloadPrepared.deferred.catch((err) => cachePrepared.deferred.reject(err));
}
fileStorage = new d();
// this.downloadToDiscPromises[cacheFileName] = deferred;
// deferred.catch(noop).finally(() => {
// if(this.downloadToDiscPromises[cacheFileName] === deferred) {
// delete this.downloadToDiscPromises[cacheFileName];
// }
// });
}
fileStorage.getFile(fileName).then(async(blob: Blob) => {
// throw '';
deferred = downloadPrepared?.deferred ?? cachePrepared.deferred;
if(blob.size < size) {
if(!options.onlyCache) {
await this.delete(fileName);
}
if(downloadStorage && process) { // then have to load file again
getFile = downloadStorage.getFile.bind(downloadStorage);
}
getFile(cacheFileName).then(async(blob: Blob) => {
checkCancel();
// if(blob.size < size) {
// if(!options.onlyCache) {
// await this.delete(cacheFileName);
// checkCancel();
// }
throw false;
// throw makeError('NO_ENTRY_FOUND');
// }
if(downloadPrepared) {
const writer = downloadPrepared.writer = downloadPrepared.getWriter();
checkCancel();
const arr = await readBlobAsUint8Array(blob);
checkCancel();
await writer.write(arr);
checkCancel();
downloadPrepared.deferred.resolve(await writer.finalize());
}
deferred.resolve(blob);
if(cachePrepared) {
cachePrepared.deferred.resolve(blob);
}
}).catch(async(err: ApiError) => {
if(options.onlyCache) {
errorHandler(err);
errorHandler(null, err);
return;
}
// this.log('not cached', fileName);
const limit = options.limitPart || this.getLimitPart(size, false);
const writerPromise = fileStorage.getWriter(fileName, size || limit, mimeType);
const writer = cacheWriter = await writerPromise;
prepared.forEach((p) => {
p.writer = p.getWriter();
});
let offset: number;
const startOffset = 0;
let writePromise: CancellablePromise<void> = Promise.resolve(),
writeDeferred: CancellablePromise<void>;
// const maxRequests = 13107200 / limit; // * 100 Mb speed
const maxRequests = Infinity;
const processDownloaded = async(bytes: Uint8Array) => {
if(process) {
// const perf = performance.now();
const processed = await process(bytes, fileName);
// this.log('downloadFile process downloaded time', performance.now() - perf, mimeType, process);
return processed;
}
const isWebFile = location._ === 'inputWebFileLocation';
const requestPart = (isWebFile ? this.requestWebFilePart : this.requestFilePart).bind(this);
return bytes;
};
const r = location._ === 'inputWebFileLocation' ? this.requestWebFilePart.bind(this) : this.requestFilePart.bind(this);
if(isWebFile && this.webFileDcId === undefined) {
await this.apiManager.getConfig();
checkCancel();
}
const delayed: Delayed[] = [];
offset = startOffset;
do {
writeDeferred = deferredPromise<void>();
delayed.push({offset, writePromise, writeDeferred});
writePromise = writeDeferred;
offset += limit;
} while(offset < size);
const delayed = this.allocateDeferredPromises(0, size, limitPart);
const progress: Progress = {done: 0, offset, total: size, fileName};
const progress: Progress = {done: 0, offset: 0, total: size, fileName};
const dispatchProgress = () => {
progress.done = done;
deferred.notify?.(progress);
try {
checkCancel();
progress.done = done;
this.rootScope.dispatchEvent('download_progress', progress);
} catch(err) {}
};
const throttledDispatchProgress = throttle(dispatchProgress, 50, true);
let done = 0;
const superpuper = async() => {
// if(!delayed.length) return;
const {offset, writePromise, writeDeferred} = delayed.shift();
try {
checkCancel();
// @ts-ignore
const result = await r(dcId, location as any, offset, limit, id, options.queueId, checkCancel);
const requestPerf = performance.now();
const result = await requestPart(dcId, location as any, offset, limitPart, id, options.queueId, checkCancel);
const requestTime = performance.now() - requestPerf;
const bytes = result.bytes;
@ -603,7 +660,7 @@ export class ApiFileManager extends AppManager { @@ -603,7 +660,7 @@ export class ApiFileManager extends AppManager {
const byteLength = bytes.byteLength;
this.debug && this.log('downloadFile requestFilePart result:', fileName, result);
const isFinal = (offset + limit) >= size || !byteLength;
const isFinal = (offset + limitPart) >= size || !byteLength;
if(byteLength) {
done += byteLength;
@ -613,68 +670,65 @@ export class ApiFileManager extends AppManager { @@ -613,68 +670,65 @@ export class ApiFileManager extends AppManager {
throttledDispatchProgress();
}
const writeQueuePerf = performance.now();
await writePromise;
checkCancel();
const writeQueueTime = performance.now() - writeQueuePerf;
// const perf = performance.now();
await writer.write(bytes, offset);
const perf = performance.now();
await Promise.all(prepared.map(({writer}) => writer?.write(bytes, offset)));
checkCancel();
// downloadId && this.log('write time', performance.now() - perf);
// downloadId && this.log('write time', performance.now() - perf, 'request time', requestTime, 'queue time', writeQueueTime);
}
if(isFinal && process) {
const bytes = writer.getParts();
const processedResult = await processDownloaded(bytes);
checkCancel();
const promises = prepared
.filter(({writer}) => writer?.getParts && writer.replaceParts)
.map(async({writer}) => {
const bytes = writer.getParts();
const processedResult = await process(bytes, cacheFileName);
writer.replaceParts(processedResult);
});
writer.replaceParts(processedResult);
await Promise.all(promises);
checkCancel();
}
writeDeferred.resolve();
if(isFinal) {
resolved = true;
const realSize = size || byteLength;
if(!size) {
writer.trim(realSize);
if(!size || byteLength < size) {
prepared.forEach(({writer}) => writer?.trim?.(realSize));
}
deferred.resolve(await writer.finalize(realSize <= MAX_FILE_SAVE_SIZE));
const saveToStorage = realSize <= MAX_FILE_SAVE_SIZE;
prepared.forEach((item) => {
const {deferred, writer} = item;
if(deferred.isFulfilled || deferred.isRejected || !writer) {
return;
}
const result = writer.finalize(saveToStorage);
deferred.resolve(result);
});
}
} catch(err) {
errorHandler(err as ApiError);
errorHandler(null, err as ApiError);
}
};
for(let i = 0, length = Math.min(maxRequests, delayed.length); i < length; ++i) {
superpuper();
}
});
}).catch(noop);
const checkCancel = () => {
if(error) {
throw error;
if(getErrorsCount() === prepared.length) {
throw prepared[0].error;
}
};
deferred.cancel = () => {
if(!error && !resolved) {
const error = makeError('DOWNLOAD_CANCELED');
errorHandler(error);
}
};
deferred.notify = (progress: Progress) => {
this.rootScope.dispatchEvent('download_progress', progress);
};
this.downloadPromises[fileName] = deferred;
deferred.catch(noop).finally(() => {
delete this.downloadPromises[fileName];
});
return deferred;
}
@ -736,23 +790,14 @@ export class ApiFileManager extends AppManager { @@ -736,23 +790,14 @@ export class ApiFileManager extends AppManager {
}
public upload({file, fileName}: {file: Blob | File, fileName?: string}) {
const fileSize = file.size,
isBigFile = fileSize >= 10485760;
let canceled = false,
resolved = false,
doneParts = 0;
const partSize = this.getLimitPart(fileSize, true);
fileName ||= getFileNameForUpload(file);
const fileSize = file.size;
const isBigFile = fileSize >= 10485760;
const partSize = this.getLimitPart(fileSize, true);
const activeDelta = this.getDelta(partSize);
const totalParts = Math.ceil(fileSize / partSize);
const fileId = randomLong();
let _part = 0;
const resultInputFile: InputFile = {
_: isBigFile ? 'inputFileBig' : 'inputFile',
id: fileId as any,
@ -767,6 +812,7 @@ export class ApiFileManager extends AppManager { @@ -767,6 +812,7 @@ export class ApiFileManager extends AppManager {
return deferred;
}
let canceled = false, resolved = false;
let errorHandler = (error: ApiError) => {
if(error?.type !== 'UPLOAD_CANCELED') {
this.log.error('Up Error', error);
@ -774,74 +820,47 @@ export class ApiFileManager extends AppManager { @@ -774,74 +820,47 @@ export class ApiFileManager extends AppManager {
deferred.reject(error);
canceled = true;
errorHandler = () => {};
errorHandler = noop;
};
const method = isBigFile ? 'upload.saveBigFilePart' : 'upload.saveFilePart';
const id = this.tempId++;
const self = this;
function* generator() {
let _part = 0, doneParts = 0;
for(let offset = 0; offset < fileSize; offset += partSize) {
const part = _part++; // 0, 1
yield self.downloadRequest('upload', id, () => {
yield self.downloadRequest('upload', id, async() => {
checkCancel();
const blob = file.slice(offset, offset + partSize);
const buffer = await readBlobAsArrayBuffer(blob);
checkCancel();
return readBlobAsArrayBuffer(blob).then((buffer) => {
self.debug && self.log('Upload file part, isBig:', isBigFile, part, buffer.byteLength, new Uint8Array(buffer).length, new Uint8Array(buffer).slice().length);
return self.apiManager.invokeApi(method, {
file_id: fileId,
file_part: part,
file_total_parts: totalParts,
bytes: buffer
} as any, {
fileUpload: true
}).then(() => {
if(canceled) {
throw makeError('UPLOAD_CANCELED');
return;
}
self.debug && self.log('Upload file part, isBig:', isBigFile, part, buffer.byteLength, new Uint8Array(buffer).length, new Uint8Array(buffer).slice().length);
++doneParts;
const progress: Progress = {done: doneParts * partSize, offset, total: fileSize, fileName};
deferred.notify(progress);
/* const u = new Uint8Array(buffer.byteLength);
for(let i = 0; i < u.length; ++i) {
//u[i] = Math.random() * 255 | 0;
u[i] = 0;
if(doneParts >= totalParts) {
deferred.resolve(resultInputFile);
resolved = true;
}
buffer = u.buffer; */
/* setTimeout(() => {
doneParts++;
uploadResolve();
//////this.log('Progress', doneParts * partSize / fileSize);
self.log('done part', part, doneParts);
deferred.notify({done: doneParts * partSize, total: fileSize});
if(doneParts >= totalParts) {
deferred.resolve(resultInputFile);
resolved = true;
}
}, 1250);
return; */
return self.apiManager.invokeApi(method, {
file_id: fileId,
file_part: part,
file_total_parts: totalParts,
bytes: buffer/* new Uint8Array(buffer) */
} as any, {
// startMaxLength: partSize + 256,
fileUpload: true
}).then(() => {
if(canceled) {
return;
}
++doneParts;
const progress: Progress = {done: doneParts * partSize, offset, total: fileSize, fileName};
deferred.notify(progress);
if(doneParts >= totalParts) {
deferred.resolve(resultInputFile);
resolved = true;
}
}, errorHandler);
});
}, errorHandler);
}, activeDelta).catch(errorHandler);
}
}
@ -855,14 +874,16 @@ export class ApiFileManager extends AppManager { @@ -855,14 +874,16 @@ export class ApiFileManager extends AppManager {
};
const maxRequests = Infinity;
// const maxRequests = 10;
/* for(let i = 0; i < 10; ++i) {
process();
} */
for(let i = 0, length = Math.min(maxRequests, totalParts); i < length; ++i) {
process();
}
const checkCancel = () => {
if(canceled) {
throw makeError('UPLOAD_CANCELED');
}
};
deferred.cancel = () => {
if(!canceled && !resolved) {
canceled = true;
@ -875,7 +896,9 @@ export class ApiFileManager extends AppManager { @@ -875,7 +896,9 @@ export class ApiFileManager extends AppManager {
};
deferred.finally(() => {
delete this.uploadPromises[fileName];
if(this.uploadPromises[fileName] === deferred) {
delete this.uploadPromises[fileName];
}
});
return this.uploadPromises[fileName] = deferred;

72
src/lib/mtproto/api_methods.ts

@ -6,7 +6,7 @@ @@ -6,7 +6,7 @@
import ctx from '../../environment/ctx';
import {ignoreRestrictionReasons} from '../../helpers/restrictions';
import {MethodDeclMap, User} from '../../layer';
import {Config, MethodDeclMap, User} from '../../layer';
import {InvokeApiOptions} from '../../types';
import {AppManager} from '../appManagers/manager';
import {MTAppConfig} from './appConfig';
@ -43,8 +43,8 @@ export default abstract class ApiManagerMethods extends AppManager { @@ -43,8 +43,8 @@ export default abstract class ApiManagerMethods extends AppManager {
}
} = {};
private config: Config;
private appConfig: MTAppConfig;
private getAppConfigPromise: Promise<MTAppConfig>;
constructor() {
super();
@ -140,7 +140,7 @@ export default abstract class ApiManagerMethods extends AppManager { @@ -140,7 +140,7 @@ export default abstract class ApiManagerMethods extends AppManager {
processResult: (response: MethodDeclMap[T]['res']) => R,
processError?: (error: ApiError) => any,
params?: MethodDeclMap[T]['req'],
options?: InvokeApiOptions & {cacheKey?: string}
options?: InvokeApiOptions & {cacheKey?: string, overwrite?: boolean}
}): Promise<Awaited<R>> {
o.params ??= {};
o.options ??= {};
@ -154,10 +154,32 @@ export default abstract class ApiManagerMethods extends AppManager { @@ -154,10 +154,32 @@ export default abstract class ApiManagerMethods extends AppManager {
return oldPromise;
}
const getNewPromise = () => {
const promise = map.get(cacheKey);
return promise === p ? undefined : promise;
}
const originalPromise = this.invokeApi(method, params, options);
const newPromise: Promise<Awaited<R>> = originalPromise.then(processResult, processError);
const newPromise: Promise<Awaited<R>> = originalPromise.then((result) => {
return getNewPromise() || processResult(result);
}, (error) => {
const promise = getNewPromise();
if(promise) {
return promise;
}
if(!processError) {
throw error;
}
return processError(error);
});
const p = newPromise.finally(() => {
if(map.get(cacheKey) !== p) {
return;
}
map.delete(cacheKey);
if(!map.size) {
delete cache[method];
@ -226,24 +248,38 @@ export default abstract class ApiManagerMethods extends AppManager { @@ -226,24 +248,38 @@ export default abstract class ApiManagerMethods extends AppManager {
}
}
public getConfig() {
return this.invokeApiCacheable('help.getConfig');
public getConfig(overwrite?: boolean) {
if(this.config && !overwrite) {
return this.config;
}
return this.invokeApiSingleProcess({
method: 'help.getConfig',
params: {},
processResult: (config) => {
this.config = config;
this.rootScope.dispatchEvent('config', config);
return config;
},
options: {overwrite}
});
}
public getAppConfig(overwrite?: boolean) {
if(this.appConfig && !overwrite) return this.appConfig;
if(this.getAppConfigPromise && !overwrite) return this.getAppConfigPromise;
const promise: Promise<MTAppConfig> = this.getAppConfigPromise = this.invokeApi('help.getAppConfig').then((config: MTAppConfig) => {
if(this.getAppConfigPromise !== promise) {
return this.getAppConfigPromise;
}
if(this.appConfig && !overwrite) {
return this.appConfig;
}
this.appConfig = config;
ignoreRestrictionReasons(config.ignore_restriction_reasons ?? []);
this.rootScope.dispatchEvent('app_config', config);
return config;
return this.invokeApiSingleProcess({
method: 'help.getAppConfig',
params: {},
processResult: (config: MTAppConfig) => {
this.appConfig = config;
ignoreRestrictionReasons(config.ignore_restriction_reasons ?? []);
this.rootScope.dispatchEvent('app_config', config);
return config;
},
options: {overwrite}
});
return promise;
}
}

2
src/lib/mtproto/dcConfigurator.ts

@ -51,7 +51,7 @@ export function getTelegramConnectionSuffix(connectionType: ConnectionType) { @@ -51,7 +51,7 @@ export function getTelegramConnectionSuffix(connectionType: ConnectionType) {
// #if MTPROTO_HAS_WS
export function constructTelegramWebSocketUrl(dcId: DcId, connectionType: ConnectionType, premium?: boolean) {
const suffix = getTelegramConnectionSuffix(connectionType);
const path = connectionType !== 'client' ? 'apiws' + (premium ? PREMIUM_SUFFIX : TEST_SUFFIX) : ('apiws' + TEST_SUFFIX);
const path = connectionType !== 'client' ? 'apiws' + TEST_SUFFIX + (premium ? PREMIUM_SUFFIX : '') : ('apiws' + TEST_SUFFIX);
const chosenServer = `wss://${App.suffix.toLowerCase()}ws${dcId}${suffix}.web.telegram.org/${path}`;
return chosenServer;

4
src/lib/mtproto/transports/websocket.ts

@ -86,8 +86,8 @@ export default class Socket extends EventListenerBase<{ @@ -86,8 +86,8 @@ export default class Socket extends EventListenerBase<{
this.close();
};
private handleClose = () => {
this.log('closed'/* , event, this.pending, this.ws.bufferedAmount */);
private handleClose = (e?: CloseEvent) => {
this.log('closed', e/* , this.pending, this.ws.bufferedAmount */);
this.removeListeners();
this.dispatchEvent('close');

3
src/lib/rootScope.ts

@ -4,7 +4,7 @@ @@ -4,7 +4,7 @@
* https://github.com/morethanwords/tweb/blob/master/LICENSE
*/
import type {Message, StickerSet, Update, NotifyPeer, PeerNotifySettings, PollResults, Poll, WebPage, GroupCall, GroupCallParticipant, ReactionCount, MessagePeerReaction, PhoneCall} from '../layer';
import type {Message, StickerSet, Update, NotifyPeer, PeerNotifySettings, PollResults, Poll, WebPage, GroupCall, GroupCallParticipant, ReactionCount, MessagePeerReaction, PhoneCall, Config} from '../layer';
import type {AppMessagesManager, Dialog, MessagesStorageKey, MyMessage} from './appManagers/appMessagesManager';
import type {MyDialogFilter} from './storages/filters';
import type {Folder} from './storages/dialogs';
@ -143,6 +143,7 @@ export type BroadcastEvents = { @@ -143,6 +143,7 @@ export type BroadcastEvents = {
'premium_toggle': boolean,
'config': Config,
'app_config': MTAppConfig
};

162
src/lib/serviceWorker/download.ts

@ -0,0 +1,162 @@ @@ -0,0 +1,162 @@
/*
* https://github.com/morethanwords/tweb
* Copyright (C) 2019-2021 Eduard Kuzmenko
* https://github.com/morethanwords/tweb/blob/master/LICENSE
*/
import type {ServiceDownloadTaskPayload} from './serviceMessagePort';
import type ServiceMessagePort from './serviceMessagePort';
import deferredPromise, {CancellablePromise} from '../../helpers/cancellablePromise';
import makeError from '../../helpers/makeError';
import pause from '../../helpers/schedulers/pause';
type DownloadType = Uint8Array;
type DownloadItem = ServiceDownloadTaskPayload & {
// transformStream: TransformStream<DownloadType, DownloadType>,
readableStream: ReadableStream<DownloadType>,
// writableStream: WritableStream<DownloadType>,
// writer: WritableStreamDefaultWriter<DownloadType>,
// controller: TransformStreamDefaultController<DownloadType>,
controller: ReadableStreamController<Uint8Array>,
promise: CancellablePromise<void>,
// downloadPromise: Promise<void>,
used?: boolean
};
const downloadMap: Map<string, DownloadItem> = new Map();
const DOWNLOAD_ERROR = makeError('UNKNOWN');
const DOWNLOAD_TEST = false;
type A = Parameters<ServiceMessagePort<false>['addMultipleEventsListeners']>[0];
const events: A = {
download: (payload) => {
const {id} = payload;
if(downloadMap.has(id)) {
return Promise.reject(DOWNLOAD_ERROR);
}
// const y = (20 * 1024 * 1024) / payload.limitPart;
// const strategy = new ByteLengthQueuingStrategy({highWaterMark: y});
// let controller: TransformStreamDefaultController<DownloadType>;
const strategy = new CountQueuingStrategy({highWaterMark: 1});
// const transformStream = new TransformStream<DownloadType, DownloadType>(/* {
// start: (_controller) => controller = _controller,
// }, */undefined, strategy, strategy);
// const {readable, writable} = transformStream;
// const writer = writable.getWriter();
const promise = deferredPromise<void>();
promise.then(() => {
setTimeout(() => {
downloadMap.delete(id);
}, 5e3);
}, () => {
downloadMap.delete(id);
});
// writer.closed.then(promise.resolve, promise.reject);
let controller: ReadableStreamController<any>;
const readable = new ReadableStream({
start: (_controller) => {
controller = _controller;
},
cancel: (reason) => {
promise.reject(DOWNLOAD_ERROR);
}
}, strategy);
// writer.closed.catch(noop).finally(() => {
// log.error('closed writer');
// onEnd();
// });
// const downloadPromise = writer.closed.catch(() => {throw DOWNLOAD_ERROR;});
const item: DownloadItem = {
...payload,
// transformStream,
readableStream: readable,
// writableStream: writable,
// writer,
// downloadPromise,
promise,
controller
};
downloadMap.set(id, item);
// return downloadPromise;
return promise.catch(() => {throw DOWNLOAD_ERROR});
},
downloadChunk: ({id, chunk}) => {
const item = downloadMap.get(id);
if(!item) {
return Promise.reject();
}
// return item.controller.enqueue(chunk);
// return item.writer.write(chunk);
return item.controller.enqueue(chunk);
},
downloadFinalize: (id) => {
const item = downloadMap.get(id);
if(!item) {
return Promise.reject();
}
item.promise.resolve();
// return item.controller.terminate();
// return item.writer.close();
return item.controller.close();
},
downloadCancel: (id) => {
const item = downloadMap.get(id);
if(!item) {
return;
}
item.promise.reject();
// return item.controller.error();
// return item.writer.abort();
return item.controller.error();
}
};
export default function handleDownload(serviceMessagePort: ServiceMessagePort<false>) {
serviceMessagePort.addMultipleEventsListeners(events);
return {
onDownloadFetch,
onClosedWindows: cancelAllDownloads
};
}
function onDownloadFetch(event: FetchEvent, params: string) {
event.respondWith(pause(100).then(() => {
const item = downloadMap.get(params);
if(!item || (item.used && !DOWNLOAD_TEST)) {
return;
}
item.used = true;
const stream = item.readableStream;
const response = new Response(stream, {headers: item.headers});
return response;
}));
// event.respondWith(response);
}
function cancelAllDownloads() {
if(downloadMap.size) {
for(const [id, item] of downloadMap) {
// item.writer.abort().catch(noop);
item.controller.error();
}
}
}

124
src/lib/serviceWorker/index.service.ts

@ -14,12 +14,11 @@ import onStreamFetch from './stream'; @@ -14,12 +14,11 @@ import onStreamFetch from './stream';
import {closeAllNotifications, onPing} from './push';
import CacheStorageController from '../files/cacheStorage';
import {IS_SAFARI} from '../../environment/userAgent';
import ServiceMessagePort, {ServiceDownloadTaskPayload} from './serviceMessagePort';
import ServiceMessagePort from './serviceMessagePort';
import listenMessagePort from '../../helpers/listenMessagePort';
import {getWindowClients} from '../../helpers/context';
import {MessageSendPort} from '../mtproto/superMessagePort';
import noop from '../../helpers/noop';
import makeError from '../../helpers/makeError';
import handleDownload from './download';
export const log = logger('SW', LogTypes.Error | LogTypes.Debug | LogTypes.Log | LogTypes.Warn);
const ctx = self as any as ServiceWorkerGlobalScope;
@ -52,19 +51,6 @@ const onWindowConnected = (source: WindowClient) => { @@ -52,19 +51,6 @@ const onWindowConnected = (source: WindowClient) => {
connectedWindows.add(source.id);
};
type DownloadType = Uint8Array;
type DownloadItem = ServiceDownloadTaskPayload & {
transformStream: TransformStream<DownloadType, DownloadType>,
readableStream: ReadableStream<DownloadType>,
writableStream: WritableStream<DownloadType>,
writer: WritableStreamDefaultWriter<DownloadType>,
// controller: TransformStreamDefaultController<DownloadType>,
// promise: CancellablePromise<void>,
used?: boolean
};
const downloadMap: Map<string, DownloadItem> = new Map();
const DOWNLOAD_ERROR = makeError('UNKNOWN');
export const serviceMessagePort = new ServiceMessagePort<false>();
serviceMessagePort.addMultipleEventsListeners({
notificationsClear: closeAllNotifications,
@ -79,86 +65,14 @@ serviceMessagePort.addMultipleEventsListeners({ @@ -79,86 +65,14 @@ serviceMessagePort.addMultipleEventsListeners({
hello: (payload, source) => {
onWindowConnected(source as any as WindowClient);
},
download: (payload) => {
const {id} = payload;
if(downloadMap.has(id)) {
return;
}
// const writableStrategy = new ByteLengthQueuingStrategy({highWaterMark: 1024 * 1024});
// let controller: TransformStreamDefaultController<DownloadType>;
const transformStream = new TransformStream<DownloadType, DownloadType>(/* {
start: (_controller) => controller = _controller,
}, {
highWaterMark: 1,
size: (chunk) => chunk.byteLength
}, new CountQueuingStrategy({highWaterMark: 4}) */);
const {readable, writable} = transformStream;
const writer = writable.getWriter();
// const promise = deferredPromise<void>();
// promise.catch(noop).finally(() => {
// downloadMap.delete(id);
// });
// writer.closed.then(promise.resolve, promise.reject);
writer.closed.catch(noop).finally(() => {
log.error('closed writer');
downloadMap.delete(id);
});
const item: DownloadItem = {
...payload,
transformStream,
readableStream: readable,
writableStream: writable,
writer
// promise,
// controller
};
downloadMap.set(id, item);
return writer.closed.catch(() => {throw DOWNLOAD_ERROR;});
// return promise;
},
downloadChunk: ({id, chunk}) => {
const item = downloadMap.get(id);
if(!item) {
return Promise.reject();
}
// return item.controller.enqueue(chunk);
return item.writer.write(chunk);
},
downloadFinalize: (id) => {
const item = downloadMap.get(id);
if(!item) {
return Promise.reject();
}
// item.promise.resolve();
// return item.controller.terminate();
return item.writer.close();
},
downloadCancel: (id) => {
const item = downloadMap.get(id);
if(!item) {
return;
}
// item.promise.reject();
// return item.controller.error();
return item.writer.abort();
}
});
const {
onDownloadFetch,
onClosedWindows: onDownloadClosedWindows
} = handleDownload(serviceMessagePort);
// * service worker can be killed, so won't get 'hello' event
getWindowClients().then((windowClients) => {
log(`got ${windowClients.length} windows from the start`);
@ -184,11 +98,7 @@ listenMessagePort(serviceMessagePort, undefined, (source) => { @@ -184,11 +98,7 @@ listenMessagePort(serviceMessagePort, undefined, (source) => {
_mtprotoMessagePort = undefined;
}
if(downloadMap.size) {
for(const [id, item] of downloadMap) {
item.writer.abort().catch(noop);
}
}
onDownloadClosedWindows();
}
});
// #endif
@ -216,14 +126,7 @@ const onFetch = (event: FetchEvent): void => { @@ -216,14 +126,7 @@ const onFetch = (event: FetchEvent): void => {
}
case 'download': {
const item = downloadMap.get(params);
if(!item || item.used) {
break;
}
item.used = true;
const response = new Response(item.transformStream.readable, {headers: item.headers});
event.respondWith(response);
onDownloadFetch(event, params);
break;
}
}
@ -231,7 +134,8 @@ const onFetch = (event: FetchEvent): void => { @@ -231,7 +134,8 @@ const onFetch = (event: FetchEvent): void => {
log.error('fetch error', err);
event.respondWith(new Response('', {
status: 500,
statusText: 'Internal Server Error'
statusText: 'Internal Server Error',
headers: {'Cache-Control': 'no-cache'}
}));
}
};
@ -242,13 +146,13 @@ const onChangeState = () => { @@ -242,13 +146,13 @@ const onChangeState = () => {
ctx.addEventListener('install', (event) => {
log('installing');
event.waitUntil(ctx.skipWaiting()); // Activate worker immediately
event.waitUntil(ctx.skipWaiting().then(() => log('skipped waiting'))); // Activate worker immediately
});
ctx.addEventListener('activate', (event) => {
log('activating', ctx);
event.waitUntil(ctx.caches.delete(CACHE_ASSETS_NAME));
event.waitUntil(ctx.clients.claim());
event.waitUntil(ctx.caches.delete(CACHE_ASSETS_NAME).then(() => log('cleared assets cache')));
event.waitUntil(ctx.clients.claim().then(() => log('claimed clients')));
});
// ctx.onerror = (error) => {

3
src/lib/serviceWorker/serviceMessagePort.ts

@ -28,9 +28,8 @@ export type ServiceRequestFilePartTaskPayload = { @@ -28,9 +28,8 @@ export type ServiceRequestFilePartTaskPayload = {
};
export type ServiceDownloadTaskPayload = {
fileName: string,
headers: any,
id: string,
id: string
};
export type ServiceEvent = {

4
src/scss/partials/_document.scss

@ -23,6 +23,10 @@ @@ -23,6 +23,10 @@
line-height: 1;
text-align: center;
html.is-safari & {
-webkit-mask-image: -webkit-radial-gradient(circle, white 100%, black 100%); // fix safari overflow
}
&-text {
opacity: 0;

Loading…
Cancel
Save