From 408db34135ac54c0571beb13c6282e8cfde917bc Mon Sep 17 00:00:00 2001 From: Eduard Kuzmenko Date: Thu, 21 Apr 2022 18:55:00 +0300 Subject: [PATCH] Use blurred canvas for thumbnails --- src/components/appMediaViewerBase.ts | 2 +- src/components/audio.ts | 2 +- src/components/sidebarLeft/tabs/background.ts | 6 +- src/components/wrappers.ts | 15 +- src/environment/canvasFilterSupport.ts | 3 + src/helpers/blur.ts | 132 +++++++++--------- src/helpers/bytes/bytesToDataURL.ts | 3 + src/helpers/dom/renderImageFromUrl.ts | 4 +- src/helpers/heavyQueue.ts | 28 ++-- src/lib/appManagers/appDocsManager.ts | 5 +- src/lib/appManagers/appPhotosManager.ts | 21 +-- src/scss/style.scss | 6 + 12 files changed, 125 insertions(+), 102 deletions(-) create mode 100644 src/environment/canvasFilterSupport.ts create mode 100644 src/helpers/bytes/bytesToDataURL.ts diff --git a/src/components/appMediaViewerBase.ts b/src/components/appMediaViewerBase.ts index b24d122b..56acb327 100644 --- a/src/components/appMediaViewerBase.ts +++ b/src/components/appMediaViewerBase.ts @@ -1292,7 +1292,7 @@ export default class AppMediaViewerBase< const size = appPhotosManager.setAttachmentSize(media, container, maxWidth, maxHeight, mediaSizes.isMobile ? false : true, undefined, !!(isDocument && media.w && media.h)).photoSize; if(useContainerAsTarget) { const cacheContext = appDownloadManager.getCacheContext(media, size.type); - let img: HTMLImageElement; + let img: HTMLImageElement | HTMLCanvasElement; if(cacheContext.downloaded) { img = new Image(); img.src = cacheContext.url; diff --git a/src/components/audio.ts b/src/components/audio.ts index 2e20ccde..19eccf0e 100644 --- a/src/components/audio.ts +++ b/src/components/audio.ts @@ -532,7 +532,7 @@ export default class AudioElement extends HTMLElement { }; if(doc.thumbs?.length) { - const imgs: HTMLImageElement[] = []; + const imgs: HTMLElement[] = []; const wrapped = wrapPhoto({ photo: doc, message: null, diff --git a/src/components/sidebarLeft/tabs/background.ts b/src/components/sidebarLeft/tabs/background.ts index 6b710c63..58972fc9 100644 --- a/src/components/sidebarLeft/tabs/background.ts +++ b/src/components/sidebarLeft/tabs/background.ts @@ -425,14 +425,14 @@ export default class AppBackgroundTab extends SliderSuperTab { const cacheContext = appDownloadManager.getCacheContext(doc); if(background.blur) { setTimeout(() => { - blur(cacheContext.url, 12, 4) - .then(url => { + const {canvas, promise} = blur(cacheContext.url, 12, 4) + promise.then(() => { if(!middleware()) { deferred.resolve(); return; } - onReady(url); + onReady(canvas.toDataURL()); }); }, 200); } else { diff --git a/src/components/wrappers.ts b/src/components/wrappers.ts index 18d3789b..90470fb7 100644 --- a/src/components/wrappers.ts +++ b/src/components/wrappers.ts @@ -625,7 +625,7 @@ export function wrapDocument({message, withTime, fontWeight, voiceAsMusic, showS if((doc.thumbs?.length || (message.pFlags.is_outgoing && cacheContext.url && doc.type === 'photo'))/* && doc.mime_type !== 'image/gif' */) { docDiv.classList.add('document-with-thumb'); - let imgs: HTMLImageElement[] = []; + let imgs: (HTMLImageElement | HTMLCanvasElement)[] = []; // ! WARNING, use thumbs for check when thumb will be generated for media if(message.pFlags.is_outgoing && ['photo', 'video'].includes(doc.type)) { icoDiv.innerHTML = ``; @@ -887,7 +887,7 @@ export function wrapPhoto({photo, message, container, boxWidth, boxHeight, withT let isFit = true; let loadThumbPromise: Promise = Promise.resolve(); - let thumbImage: HTMLImageElement; + let thumbImage: HTMLImageElement | HTMLCanvasElement; let image: HTMLImageElement; let cacheContext: ThumbCache; const isGif = photo._ === 'document' && photo.mime_type === 'image/gif' && !size; @@ -1000,8 +1000,10 @@ export function wrapPhoto({photo, message, container, boxWidth, boxHeight, withT if(middleware && !middleware()) return Promise.resolve(); if(blurAfter) { - return blur(cacheContext.url, 12).then(url => { - return renderOnLoad(url); + const result = blur(cacheContext.url, 12); + return result.promise.then(() => { + // image = result.canvas; + return renderOnLoad(result.canvas.toDataURL()); }); } @@ -1069,12 +1071,13 @@ export function wrapPhoto({photo, message, container, boxWidth, boxHeight, withT }; } -export function renderImageWithFadeIn(container: HTMLElement, +export function renderImageWithFadeIn( + container: HTMLElement, image: HTMLImageElement, url: string, needFadeIn: boolean, aspecter = container, - thumbImage?: HTMLImageElement + thumbImage?: HTMLElement ) { if(needFadeIn) { image.classList.add('fade-in'); diff --git a/src/environment/canvasFilterSupport.ts b/src/environment/canvasFilterSupport.ts new file mode 100644 index 00000000..3ab97bf3 --- /dev/null +++ b/src/environment/canvasFilterSupport.ts @@ -0,0 +1,3 @@ +const IS_CANVAS_FILTER_SUPPORTED = 'filter' in (document.createElement('canvas').getContext('2d') || {}); + +export default IS_CANVAS_FILTER_SUPPORTED; \ No newline at end of file diff --git a/src/helpers/blur.ts b/src/helpers/blur.ts index b1ff8fdc..ba3a13c1 100644 --- a/src/helpers/blur.ts +++ b/src/helpers/blur.ts @@ -6,14 +6,14 @@ import type fastBlur from '../vendor/fastBlur'; import addHeavyTask from './heavyQueue'; +import IS_CANVAS_FILTER_SUPPORTED from '../environment/canvasFilterSupport'; const RADIUS = 2; const ITERATIONS = 2; -const isFilterAvailable = 'filter' in (document.createElement('canvas').getContext('2d') || {}); let requireBlurPromise: Promise; let fastBlurFunc: typeof fastBlur; -if(!isFilterAvailable) { +if(!IS_CANVAS_FILTER_SUPPORTED) { requireBlurPromise = import('../vendor/fastBlur').then(m => { fastBlurFunc = m.default; }); @@ -21,80 +21,80 @@ if(!isFilterAvailable) { requireBlurPromise = Promise.resolve(); } -function processBlurNext(img: HTMLImageElement, radius: number, iterations: number) { - return new Promise((resolve) => { - const canvas = document.createElement('canvas'); - canvas.width = img.width; - canvas.height = img.height; - - const ctx = canvas.getContext('2d', {alpha: false}); - if(isFilterAvailable) { - ctx.filter = `blur(${radius}px)`; - ctx.drawImage(img, -radius * 2, -radius * 2, canvas.width + radius * 4, canvas.height + radius * 4); - } else { - ctx.drawImage(img, 0, 0); - fastBlurFunc(ctx, 0, 0, canvas.width, canvas.height, radius, iterations); - } - - resolve(canvas.toDataURL()); - /* if(DEBUG) { - console.log(`[blur] end, radius: ${radius}, iterations: ${iterations}, time: ${performance.now() - perf}`); - } */ +function processBlurNext( + img: HTMLImageElement, + radius: number, + iterations: number, + canvas: HTMLCanvasElement = document.createElement('canvas') +) { + canvas.width = img.width; + canvas.height = img.height; - /* canvas.toBlob(blob => { - resolve(URL.createObjectURL(blob)); - - if(DEBUG) { - console.log(`[blur] end, radius: ${radius}, iterations: ${iterations}, time: ${performance.now() - perf}`); - } - }); */ - }); + const ctx = canvas.getContext('2d', {alpha: false}); + if(IS_CANVAS_FILTER_SUPPORTED) { + ctx.filter = `blur(${radius}px)`; + ctx.drawImage(img, -radius * 2, -radius * 2, canvas.width + radius * 4, canvas.height + radius * 4); + } else { + ctx.drawImage(img, 0, 0); + fastBlurFunc(ctx, 0, 0, canvas.width, canvas.height, radius, iterations); + } + + return canvas; } -const blurPromises: Map> = new Map(); -const CACHE_SIZE = 1000; +type CacheValue = {canvas: HTMLCanvasElement, promise: Promise}; +const cache: Map = new Map(); +const CACHE_SIZE = 150; export default function blur(dataUri: string, radius: number = RADIUS, iterations: number = ITERATIONS) { if(!dataUri) { - console.error('no dataUri for blur', dataUri); - return Promise.resolve(dataUri); + throw 'no dataUri for blur: ' + dataUri; } - if(blurPromises.size > CACHE_SIZE) { - blurPromises.clear(); + if(cache.size > CACHE_SIZE) { + cache.clear(); } - - if(blurPromises.has(dataUri)) return blurPromises.get(dataUri); - const promise = new Promise((resolve) => { - //return resolve(dataUri); - requireBlurPromise.then(() => { - const img = new Image(); - img.onload = () => { - if(isFilterAvailable) { - processBlurNext(img, radius, iterations).then(resolve); - } else { - addHeavyTask({ - items: [[img, radius, iterations]], - context: null, - process: processBlurNext - }, 'unshift').then(results => { - resolve(results[0]); - }); - } - }; - img.src = dataUri; - /* addHeavyTask({ - items: [[dataUri, radius, iterations]], - context: null, - process: processBlur - }, 'unshift').then(results => { - resolve(results[0]); - }); */ + const canvas = document.createElement('canvas'); + canvas.className = 'canvas-thumbnail'; + + let cached = cache.get(dataUri); + if(!cached) { + const promise: CacheValue['promise'] = new Promise((resolve) => { + //return resolve(dataUri); + requireBlurPromise.then(() => { + const img = new Image(); + img.onload = () => { + // if(IS_CANVAS_FILTER_SUPPORTED) { + // resolve(processBlurNext(img, radius, iterations)); + // } else { + const promise = addHeavyTask({ + items: [[img, radius, iterations, canvas]], + context: null, + process: processBlurNext + }, 'unshift'); + + promise.then(() => { + resolve(); + }); + // } + }; + img.src = dataUri; + }); }); - }); - - blurPromises.set(dataUri, promise); + + cache.set(dataUri, cached = { + canvas, + promise + }); + } else { + canvas.width = cached.canvas.width; + canvas.height = cached.canvas.height; + canvas.getContext('2d').drawImage(cached.canvas, 0, 0); + } - return promise; + return { + ...cached, + canvas + }; } diff --git a/src/helpers/bytes/bytesToDataURL.ts b/src/helpers/bytes/bytesToDataURL.ts new file mode 100644 index 00000000..536555f3 --- /dev/null +++ b/src/helpers/bytes/bytesToDataURL.ts @@ -0,0 +1,3 @@ +export default function bytesToDataURL(bytes: Uint8Array, mimeType: string = 'image/jpeg') { + return `data:${mimeType};base64,${btoa(String.fromCharCode(...bytes))}`; +} diff --git a/src/helpers/dom/renderImageFromUrl.ts b/src/helpers/dom/renderImageFromUrl.ts index 3c3db569..c34c3ac8 100644 --- a/src/helpers/dom/renderImageFromUrl.ts +++ b/src/helpers/dom/renderImageFromUrl.ts @@ -17,7 +17,7 @@ const set = (elem: HTMLElement | HTMLImageElement | SVGImageElement | HTMLVideoE export default function renderImageFromUrl( elem: HTMLElement | HTMLImageElement | SVGImageElement | HTMLVideoElement, url: string, - callback?: (err?: Event) => void, + callback?: () => void, useCache = true ) { if(!url) { @@ -61,7 +61,7 @@ export default function renderImageFromUrl( } export function renderImageFromUrlPromise(elem: Parameters[0], url: string, useCache?: boolean) { - return new Promise((resolve) => { + return new Promise((resolve) => { renderImageFromUrl(elem, url, resolve, useCache); }); } diff --git a/src/helpers/heavyQueue.ts b/src/helpers/heavyQueue.ts index f6107380..e337846f 100644 --- a/src/helpers/heavyQueue.ts +++ b/src/helpers/heavyQueue.ts @@ -7,26 +7,27 @@ import deferredPromise, { CancellablePromise } from "./cancellablePromise"; import { getHeavyAnimationPromise } from "../hooks/useHeavyAnimationCheck"; import { fastRaf } from "./schedulers"; +import { ArgumentTypes } from "../types"; -type HeavyQueue = { - items: any[], - process: (...args: any[]) => T, +type HeavyQueue> = { + items: ArgumentTypes[], + process: (...args: any[]) => ReturnType, context: any, - promise?: CancellablePromise['process']>[]> + promise?: CancellablePromise[]> }; const heavyQueue: HeavyQueue[] = []; let processingQueue = false; -export default function addHeavyTask(queue: HeavyQueue, method: 'push' | 'unshift' = 'push') { +export default function addHeavyTask>(queue: T, method: 'push' | 'unshift' = 'push') { if(!queue.items.length) { - return Promise.resolve([]); + return Promise.resolve([]) as typeof promise; } - queue.promise = deferredPromise(); + const promise = queue.promise = deferredPromise(); heavyQueue[method](queue); processHeavyQueue(); - return queue.promise; + return promise; } function processHeavyQueue() { @@ -41,23 +42,24 @@ function processHeavyQueue() { } } -function timedChunk(queue: HeavyQueue) { +function timedChunk>(queue: HeavyQueue) { if(!queue.items.length) { - queue.promise.resolve([]); + queue.promise.resolve([] as any); return Promise.resolve([]); } const todo = queue.items.slice(); - const results: T[] = []; + const results: ReturnType[] = []; - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { const f = async() => { const start = performance.now(); do { await getHeavyAnimationPromise(); const possiblePromise = queue.process.apply(queue.context, todo.shift()); - let realResult: T; + let realResult: typeof results[0]; + // @ts-ignore if(possiblePromise instanceof Promise) { try { realResult = await possiblePromise; diff --git a/src/lib/appManagers/appDocsManager.ts b/src/lib/appManagers/appDocsManager.ts index c084ef64..eb3bea9c 100644 --- a/src/lib/appManagers/appDocsManager.ts +++ b/src/lib/appManagers/appDocsManager.ts @@ -314,8 +314,9 @@ export class AppDocsManager { const cacheContext = appDownloadManager.getCacheContext(doc, thumb.type); if(!cacheContext.url) { if('bytes' in thumb) { - promise = blur(appPhotosManager.getPreviewURLFromBytes(thumb.bytes, !!doc.sticker)).then(url => { - cacheContext.url = url; + const result = blur(appPhotosManager.getPreviewURLFromBytes(thumb.bytes, !!doc.sticker)); + promise = result.promise.then(() => { + cacheContext.url = result.canvas.toDataURL(); }) as any; } else { //return this.getFileURL(doc, false, thumb); diff --git a/src/lib/appManagers/appPhotosManager.ts b/src/lib/appManagers/appPhotosManager.ts index 5c744ad2..20849875 100644 --- a/src/lib/appManagers/appPhotosManager.ts +++ b/src/lib/appManagers/appPhotosManager.ts @@ -28,6 +28,7 @@ import windowSize from "../../helpers/windowSize"; import bytesFromHex from "../../helpers/bytes/bytesFromHex"; import isObject from "../../helpers/object/isObject"; import safeReplaceArrayInObject from "../../helpers/object/safeReplaceArrayInObject"; +import bytesToDataURL from "../../helpers/bytes/bytesToDataURL"; export type MyPhoto = Photo.photo; @@ -171,8 +172,7 @@ export class AppPhotosManager { mimeType = 'image/jpeg'; } - const blob = new Blob([arr], {type: mimeType}); - return URL.createObjectURL(blob); + return bytesToDataURL(arr, mimeType); } /** @@ -210,14 +210,19 @@ export class AppPhotosManager { public getImageFromStrippedThumb(photo: MyPhoto | MyDocument, thumb: PhotoSize.photoCachedSize | PhotoSize.photoStrippedSize, useBlur: boolean) { const url = this.getPreviewURLFromThumb(photo, thumb, false); - const image = new Image(); - image.classList.add('thumbnail'); + let element: HTMLImageElement | HTMLCanvasElement, loadPromise: Promise; + if(!useBlur) { + element = new Image(); + loadPromise = renderImageFromUrlPromise(element, url); + } else { + const result = blur(url); + element = result.canvas; + loadPromise = result.promise; + } - const loadPromise = (useBlur ? blur(url) : Promise.resolve(url)).then(url => { - return renderImageFromUrlPromise(image, url); - }); + element.classList.add('thumbnail'); - return {image, loadPromise}; + return {image: element, loadPromise}; } public setAttachmentSize( diff --git a/src/scss/style.scss b/src/scss/style.scss index 1cdb63e6..d7f99510 100644 --- a/src/scss/style.scss +++ b/src/scss/style.scss @@ -1213,6 +1213,12 @@ middle-ellipsis-element { fill: rgba(0, 0, 0, .08); } +.canvas-thumbnail { + position: absolute; + width: 100%; + height: 100%; +} + .media-photo, .media-video, .media-sticker,