From 9f9827c7a66edc9f17642b57f777aad945785383 Mon Sep 17 00:00:00 2001 From: Igor Zhukov Date: Thu, 30 Oct 2014 14:20:10 +0300 Subject: [PATCH] Improved upload speed Added upload multi threads Improved web workers: added onload event Fixed unauthorised box --- app/js/controllers.js | 4 +- app/js/lib/bin_utils.js | 12 ++++++ app/js/lib/crypto_worker.js | 2 + app/js/lib/mtproto.js | 25 +++++++++++- app/js/lib/mtproto_wrapper.js | 75 ++++++++++++++++++++--------------- app/js/lib/ng_utils.js | 40 +++++++++++-------- app/js/services.js | 4 +- 7 files changed, 107 insertions(+), 55 deletions(-) diff --git a/app/js/controllers.js b/app/js/controllers.js index 47ab1d5e..713f4946 100644 --- a/app/js/controllers.js +++ b/app/js/controllers.js @@ -467,7 +467,6 @@ angular.module('myApp.controllers', ['myApp.i18n']) peersInDialogs = {}, contactsShown; - MtpApiManager.invokeApi('account.updateStatus', {offline: false}); $scope.$on('dialogs_need_more', function () { // console.log('on need more'); showMoreDialogs(); @@ -629,7 +628,8 @@ angular.module('myApp.controllers', ['myApp.i18n']) if (error.code == 401) { MtpApiManager.logOut()['finally'](function () { - $location.url('/login'); + location.hash = '/login'; + AppRuntimeManager.reload(); }); error.handled = true; } diff --git a/app/js/lib/bin_utils.js b/app/js/lib/bin_utils.js index 709c0e53..82905d44 100644 --- a/app/js/lib/bin_utils.js +++ b/app/js/lib/bin_utils.js @@ -175,6 +175,18 @@ function convertToUint8Array(bytes) { return new Uint8Array(bytes); } +function convertToByteArray(bytes) { + if (Array.isArray(bytes)) { + return bytes; + } + bytes = convertToUint8Array(bytes); + var newBytes = []; + for (var i = 0, len = bytes.length; i < len; i++) { + newBytes.push(bytes[i]); + } + return newBytes; +} + function bytesFromArrayBuffer (buffer) { var len = buffer.byteLength, byteView = new Uint8Array(buffer), diff --git a/app/js/lib/crypto_worker.js b/app/js/lib/crypto_worker.js index 4a3c9ff0..96ec3d30 100644 --- a/app/js/lib/crypto_worker.js +++ b/app/js/lib/crypto_worker.js @@ -46,3 +46,5 @@ onmessage = function (e) { postMessage({taskID: taskID, result: result}); } + +postMessage('ready'); diff --git a/app/js/lib/mtproto.js b/app/js/lib/mtproto.js index 4b0b3357..7b0f0a9e 100644 --- a/app/js/lib/mtproto.js +++ b/app/js/lib/mtproto.js @@ -988,13 +988,32 @@ angular.module('izhukov.mtproto', ['izhukov.utils']) currentTime = tsNow(), hasApiCall = false, hasHttpWait = false, + lengthOverflow = false, + singlesCount = 0, self = this; angular.forEach(this.pendingMessages, function (value, messageID) { if (!value || value >= currentTime) { if (message = self.sentMessages[messageID]) { + var messageByteLength = (message.body.byteLength || message.body.length) + 32; + if (!message.notContentRelated && + lengthOverflow) { + return; + } + if (!message.notContentRelated && + messagesByteLen && + messagesByteLen + messageByteLength > 655360) { // 640 Kb + lengthOverflow = true; + return; + } + if (message.singleInRequest) { + singlesCount++; + if (singlesCount > 1) { + return; + } + } messages.push(message); - messagesByteLen += (message.body.byteLength || message.body.length) + 32; + messagesByteLen += messageByteLength; if (message.isAPI) { hasApiCall = true; } @@ -1111,6 +1130,10 @@ angular.module('izhukov.mtproto', ['izhukov.utils']) self.toggleOffline(true); }); + + if (lengthOverflow || singlesCount > 1) { + this.sheduleRequest() + } }; MtpNetworker.prototype.getEncryptedMessage = function (bytes) { diff --git a/app/js/lib/mtproto_wrapper.js b/app/js/lib/mtproto_wrapper.js index bf6ca356..357fd99d 100644 --- a/app/js/lib/mtproto_wrapper.js +++ b/app/js/lib/mtproto_wrapper.js @@ -234,14 +234,12 @@ angular.module('izhukov.mtproto.wrapper', ['izhukov.utils', 'izhukov.mtproto']) var cachedFs = false; var cachedFsPromise = false; - var apiUploadPromise = $q.when(); var cachedSavePromises = {}; var cachedDownloadPromises = {}; var cachedDownloads = {}; var downloadPulls = {}; var downloadActives = {}; - var downloadLimit = 5; function downloadRequest(dcID, cb, activeDelta) { if (downloadPulls[dcID] === undefined) { @@ -251,7 +249,9 @@ angular.module('izhukov.mtproto.wrapper', ['izhukov.utils', 'izhukov.mtproto']) var downloadPull = downloadPulls[dcID]; var deferred = $q.defer(); downloadPull.push({cb: cb, deferred: deferred, activeDelta: activeDelta}); - downloadCheck(dcID); + setZeroTimeout(function () { + downloadCheck(dcID); + }); return deferred.promise; }; @@ -260,6 +260,7 @@ angular.module('izhukov.mtproto.wrapper', ['izhukov.utils', 'izhukov.mtproto']) function downloadCheck(dcID) { var downloadPull = downloadPulls[dcID]; + var downloadLimit = dcID == 'upload' ? 17 : 5; if (downloadActives[dcID] >= downloadLimit || !downloadPull || !downloadPull.length) { return false; @@ -507,15 +508,23 @@ angular.module('izhukov.mtproto.wrapper', ['izhukov.utils', 'izhukov.mtproto']) } function uploadFile (file) { - var fileSize = file.size, - // partSize = fileSize > 102400 ? 65536 : 4096, - // partSize = fileSize > 102400 ? 524288 : 4096, - partSize = fileSize > 102400 ? 524288 : 32768, - isBigFile = fileSize >= 10485760, - totalParts = Math.ceil(fileSize / partSize), - canceled = false, - resolved = false, - doneParts = 0; + var fileSize = file.size, + isBigFile = fileSize >= 10485760, + canceled = false, + resolved = false, + doneParts = 0, + partSize = 262144, // 256 Kb + activeDelta = 2; + + if (fileSize > 67108864) { + partSize = 524288; + activeDelta = 4; + } + else if (fileSize < 102400) { + partSize = 32768; + activeDelta = 1; + } + var totalParts = Math.ceil(fileSize / partSize); if (totalParts > 1500) { return $q.reject({type: 'FILE_TOO_BIG'}); @@ -526,6 +535,7 @@ angular.module('izhukov.mtproto.wrapper', ['izhukov.utils', 'izhukov.mtproto']) errorHandler = function (error) { // console.error('Up Error', error); deferred.reject(error); + canceled = true; errorHandler = angular.noop; }, part = 0, @@ -539,35 +549,34 @@ angular.module('izhukov.mtproto.wrapper', ['izhukov.utils', 'izhukov.mtproto']) }; - var fileReadPromise = $q.when(); - for (offset = 0; offset < fileSize; offset += partSize) { (function (offset, part) { - fileReadPromise = fileReadPromise.then(function () { - var fileReadDeferred = $q.defer(); + downloadRequest('upload', function () { + var uploadDeferred = $q.defer(); var reader = new FileReader(); var blob = file.slice(offset, offset + partSize); reader.onloadend = function (e) { - if (canceled || e.target.readyState != FileReader.DONE) { + if (canceled) { + uploadDeferred.reject(); return; } - var apiCurPromise = apiUploadPromise = apiUploadPromise.then(function () { - return MtpApiManager.invokeApi(isBigFile ? 'upload.saveBigFilePart' : 'upload.saveFilePart', { - file_id: fileID, - file_part: part, - file_total_parts: totalParts, - bytes: e.target.result - }, { - startMaxLength: partSize + 256, - fileUpload: true - }); - }, errorHandler); - - apiCurPromise.then(function (result) { + if (e.target.readyState != FileReader.DONE) { + return; + } + MtpApiManager.invokeApi(isBigFile ? 'upload.saveBigFilePart' : 'upload.saveFilePart', { + file_id: fileID, + file_part: part, + file_total_parts: totalParts, + bytes: e.target.result + }, { + startMaxLength: partSize + 256, + fileUpload: true, + singleInRequest: true + }).then(function (result) { doneParts++; - fileReadDeferred.resolve(); + uploadDeferred.resolve(); if (doneParts >= totalParts) { deferred.resolve(resultInputFile); resolved = true; @@ -580,8 +589,8 @@ angular.module('izhukov.mtproto.wrapper', ['izhukov.utils', 'izhukov.mtproto']) reader.readAsArrayBuffer(blob); - return fileReadDeferred.promise; - }); + return uploadDeferred.promise; + }, activeDelta); })(offset, part++); } diff --git a/app/js/lib/ng_utils.js b/app/js/lib/ng_utils.js index 854a4ea3..7a9e2545 100644 --- a/app/js/lib/ng_utils.js +++ b/app/js/lib/ng_utils.js @@ -484,12 +484,12 @@ angular.module('izhukov.utils', []) .service('CryptoWorker', function ($timeout, $q) { - var worker = window.Worker && new Worker('js/lib/crypto_worker.js') || false, + var webWorker = false, + naClEmbed = false, taskID = 0, awaiting = {}, webCrypto = window.crypto && (window.crypto.subtle || window.crypto.webkitSubtle) || window.msCrypto && window.msCrypto.subtle, useSha1Crypto = webCrypto && webCrypto.digest !== undefined, - aesNaClEmbed = false, finalizeTask = function (taskID, result) { var deferred = awaiting[taskID]; if (deferred !== undefined) { @@ -502,7 +502,7 @@ angular.module('izhukov.utils', []) if (navigator.mimeTypes['application/x-pnacl'] !== undefined) { var listener = $('
').appendTo($('body'))[0]; listener.addEventListener('load', function (e) { - aesNaClEmbed = listener.firstChild; + naClEmbed = listener.firstChild; console.log(dT(), 'NaCl ready'); }, true); listener.addEventListener('message', function (e) { @@ -513,13 +513,18 @@ angular.module('izhukov.utils', []) }, true); } - if (worker) { - worker.onmessage = function (e) { - finalizeTask(e.data.taskID, e.data.result); + if (window.Worker) { + var tmpWorker = new Worker('js/lib/crypto_worker.js'); + tmpWorker.onmessage = function (e) { + if (!webWorker) { + webWorker = tmpWorker; + } else { + finalizeTask(e.data.taskID, e.data.result); + } }; - worker.onerror = function(error) { + tmpWorker.onerror = function(error) { console.error('CW error', error, error.stack); - worker = false; + webWorker = false; }; } @@ -531,7 +536,7 @@ angular.module('izhukov.utils', []) params.task = task; params.taskID = taskID; - (embed || worker).postMessage(params); + (embed || webWorker).postMessage(params); taskID++; @@ -561,34 +566,35 @@ angular.module('izhukov.utils', []) }); }, aesEncrypt: function (bytes, keyBytes, ivBytes) { - if (aesNaClEmbed) { + if (naClEmbed) { return performTaskWorker('aes-encrypt', { bytes: addPadding(convertToArrayBuffer(bytes)), keyBytes: convertToArrayBuffer(keyBytes), ivBytes: convertToArrayBuffer(ivBytes) - }, aesNaClEmbed); + }, naClEmbed); } return $timeout(function () { return convertToArrayBuffer(aesEncryptSync(bytes, keyBytes, ivBytes)); }); }, aesDecrypt: function (encryptedBytes, keyBytes, ivBytes) { - if (aesNaClEmbed) { + if (naClEmbed) { return performTaskWorker('aes-decrypt', { encryptedBytes: addPadding(convertToArrayBuffer(encryptedBytes)), keyBytes: convertToArrayBuffer(keyBytes), ivBytes: convertToArrayBuffer(ivBytes) - }, aesNaClEmbed); + }, naClEmbed); } return $timeout(function () { return convertToArrayBuffer(aesDecryptSync(encryptedBytes, keyBytes, ivBytes)); }); }, factorize: function (bytes) { - if (aesNaClEmbed && bytes.length <= 8) { - return performTaskWorker('factorize', {bytes: bytes}, aesNaClEmbed); + bytes = convertToByteArray(bytes); + if (naClEmbed && bytes.length <= 8) { + return performTaskWorker('factorize', {bytes: bytes}, naClEmbed); } - if (worker) { + if (webWorker) { return performTaskWorker('factorize', {bytes: bytes}); } return $timeout(function () { @@ -596,7 +602,7 @@ angular.module('izhukov.utils', []) }); }, modPow: function (x, y, m) { - if (worker) { + if (webWorker) { return performTaskWorker('mod-pow', { x: x, y: y, diff --git a/app/js/services.js b/app/js/services.js index 1096a1aa..067b0c52 100644 --- a/app/js/services.js +++ b/app/js/services.js @@ -850,7 +850,7 @@ angular.module('myApp.services', ['myApp.i18n']) offset: offset || 0, limit: limit || 0, max_id: maxID || 0 - }).then(function (historyResult) { + }, {noErrorBox: true}).then(function (historyResult) { AppUsersManager.saveApiUsers(historyResult.users); AppChatsManager.saveApiChats(historyResult.chats); saveMessages(historyResult.messages); @@ -3130,7 +3130,7 @@ angular.module('myApp.services', ['myApp.i18n']) function attach () { MtpNetworkerFactory.setUpdatesProcessor(processUpdateMessage); - MtpApiManager.invokeApi('updates.getState', {noErrorBox: true}).then(function (stateResult) { + MtpApiManager.invokeApi('updates.getState', {}, {noErrorBox: true}).then(function (stateResult) { curState.seq = stateResult.seq; curState.pts = stateResult.pts; curState.date = stateResult.date;