improve: Mask Editor (#2171)

* renewal mask editor

* fix: ignoring keydown when 2nd open

web/extensions/core/maskeditor.js

@@ -33,6 +33,18 @@ function loadedImageToBlob(image) {
 	return blob;
 }
 
+function loadImage(imagePath) {
+	return new Promise((resolve, reject) => {
+		const image = new Image();
+
+		image.onload = function() {
+			resolve(image);
+		};
+
+		image.src = imagePath;
+	});
+}
+
 async function uploadMask(filepath, formData) {
 	await api.fetchApi('/upload/mask', {
 		method: 'POST',
@@ -50,25 +62,25 @@ async function uploadMask(filepath, formData) {
 	ClipspaceDialog.invalidatePreview();
 }
 
-function prepareRGB(image, backupCanvas, backupCtx) {
+function prepare_mask(image, maskCanvas, maskCtx) {
 	// paste mask data into alpha channel
-	backupCtx.drawImage(image, 0, 0, backupCanvas.width, backupCanvas.height);
+	maskCtx.drawImage(image, 0, 0, maskCanvas.width, maskCanvas.height);
-	const backupData = backupCtx.getImageData(0, 0, backupCanvas.width, backupCanvas.height);
+	const maskData = maskCtx.getImageData(0, 0, maskCanvas.width, maskCanvas.height);
 
-	// refine mask image
+	// invert mask
-	for (let i = 0; i < backupData.data.length; i += 4) {
+	for (let i = 0; i < maskData.data.length; i += 4) {
-		if(backupData.data[i+3] == 255)
+		if(maskData.data[i+3] == 255)
-			backupData.data[i+3] = 0;
+			maskData.data[i+3] = 0;
 		else
-			backupData.data[i+3] = 255;
+			maskData.data[i+3] = 255;
 
-		backupData.data[i] = 0;
+		maskData.data[i] = 0;
-		backupData.data[i+1] = 0;
+		maskData.data[i+1] = 0;
-		backupData.data[i+2] = 0;
+		maskData.data[i+2] = 0;
 	}
 
-	backupCtx.globalCompositeOperation = 'source-over';
+	maskCtx.globalCompositeOperation = 'source-over';
-	backupCtx.putImageData(backupData, 0, 0);
+	maskCtx.putImageData(maskData, 0, 0);
 }
 
 class MaskEditorDialog extends ComfyDialog {
@@ -184,14 +196,13 @@ class MaskEditorDialog extends ComfyDialog {
 		this.element.appendChild(bottom_panel);
 		document.body.appendChild(brush);
 
-		var brush_size_slider = this.createLeftSlider(self, "Thickness", (event) => {
+		this.brush_size_slider = this.createLeftSlider(self, "Thickness", (event) => {
 			self.brush_size = event.target.value;
 			self.updateBrushPreview(self, null, null);
 		});
 		var clearButton = this.createLeftButton("Clear",
 			() => {
 				self.maskCtx.clearRect(0, 0, self.maskCanvas.width, self.maskCanvas.height);
-				self.backupCtx.clearRect(0, 0, self.backupCanvas.width, self.backupCanvas.height);
 			});
 		var cancelButton = this.createRightButton("Cancel", () => {
 			document.removeEventListener("mouseup", MaskEditorDialog.handleMouseUp);
@@ -213,34 +224,37 @@ class MaskEditorDialog extends ComfyDialog {
 		bottom_panel.appendChild(clearButton);
 		bottom_panel.appendChild(this.saveButton);
 		bottom_panel.appendChild(cancelButton);
-		bottom_panel.appendChild(brush_size_slider);
+		bottom_panel.appendChild(this.brush_size_slider);
+
+		imgCanvas.style.position = "absolute";
+		maskCanvas.style.position = "absolute";
 
-		imgCanvas.style.position = "relative";
 		imgCanvas.style.top = "200";
 		imgCanvas.style.left = "0";
 
-		maskCanvas.style.position = "absolute";
+		maskCanvas.style.top = imgCanvas.style.top;
+		maskCanvas.style.left = imgCanvas.style.left;
 	}
 
-	show() {
+	async show() {
+		this.zoom_ratio = 1.0;
+		this.pan_x = 0;
+		this.pan_y = 0;
+
 		if(!this.is_layout_created) {
 			// layout
 			const imgCanvas = document.createElement('canvas');
 			const maskCanvas = document.createElement('canvas');
-			const backupCanvas = document.createElement('canvas');
 
 			imgCanvas.id = "imageCanvas";
 			maskCanvas.id = "maskCanvas";
-			backupCanvas.id = "backupCanvas";
 
 			this.setlayout(imgCanvas, maskCanvas);
 
 			// prepare content
 			this.imgCanvas = imgCanvas;
 			this.maskCanvas = maskCanvas;
-			this.backupCanvas = backupCanvas;
+			this.maskCtx = maskCanvas.getContext('2d', {willReadFrequently: true });
-			this.maskCtx = maskCanvas.getContext('2d');
-			this.backupCtx = backupCanvas.getContext('2d');
 
 			this.setEventHandler(maskCanvas);
 
@@ -252,6 +266,8 @@ class MaskEditorDialog extends ComfyDialog {
 			mutations.forEach(function(mutation) {
 					if (mutation.type === 'attributes' && mutation.attributeName === 'style') {
 						if(self.last_display_style && self.last_display_style != 'none' && self.element.style.display == 'none') {
+							document.removeEventListener("mouseup", MaskEditorDialog.handleMouseUp);
+							self.brush.style.display = "none";
 							ComfyApp.onClipspaceEditorClosed();
 						}
 
@@ -264,7 +280,8 @@ class MaskEditorDialog extends ComfyDialog {
 			observer.observe(this.element, config);
 		}
 
-		this.setImages(this.imgCanvas, this.backupCanvas);
+		// The keydown event needs to be reconfigured when closing the dialog as it gets removed.
+		document.addEventListener('keydown', MaskEditorDialog.handleKeyDown);
 
 		if(ComfyApp.clipspace_return_node) {
 			this.saveButton.innerText = "Save to node";
@@ -275,97 +292,157 @@ class MaskEditorDialog extends ComfyDialog {
 		this.saveButton.disabled = false;
 
 		this.element.style.display = "block";
+		this.element.style.width = "85%";
+		this.element.style.margin = "0 7.5%";
+		this.element.style.height = "100vh";
+		this.element.style.top = "50%";
+		this.element.style.left = "42%";
 		this.element.style.zIndex = 8888; // NOTE: alert dialog must be high priority.
+
+		await this.setImages(this.imgCanvas);
+
+		this.is_visible = true;
 	}
 
 	isOpened() {
 		return this.element.style.display == "block";
 	}
 
-	setImages(imgCanvas, backupCanvas) {
+	invalidateCanvas(orig_image, mask_image) {
-		const imgCtx = imgCanvas.getContext('2d');
+		this.imgCanvas.width = orig_image.width;
-		const backupCtx = backupCanvas.getContext('2d');
+		this.imgCanvas.height = orig_image.height;
+
+		this.maskCanvas.width = orig_image.width;
+		this.maskCanvas.height = orig_image.height;
+
+		let imgCtx = this.imgCanvas.getContext('2d', {willReadFrequently: true });
+		let maskCtx = this.maskCanvas.getContext('2d', {willReadFrequently: true });
+
+		imgCtx.drawImage(orig_image, 0, 0, orig_image.width, orig_image.height);
+		prepare_mask(mask_image, this.maskCanvas, maskCtx);
+	}
+
+	async setImages(imgCanvas) {
+		let self = this;
+
+		const imgCtx = imgCanvas.getContext('2d', {willReadFrequently: true });
 		const maskCtx = this.maskCtx;
 		const maskCanvas = this.maskCanvas;
 
-		backupCtx.clearRect(0,0,this.backupCanvas.width,this.backupCanvas.height);
 		imgCtx.clearRect(0,0,this.imgCanvas.width,this.imgCanvas.height);
 		maskCtx.clearRect(0,0,this.maskCanvas.width,this.maskCanvas.height);
 
 		// image load
-		const orig_image = new Image();
-		window.addEventListener("resize", () => {
-			// repositioning
-			imgCanvas.width = window.innerWidth - 250;
-			imgCanvas.height = window.innerHeight - 200;
-
-			// redraw image
-			let drawWidth = orig_image.width;
-			let drawHeight = orig_image.height;
-			if (orig_image.width > imgCanvas.width) {
-				drawWidth = imgCanvas.width;
-				drawHeight = (drawWidth / orig_image.width) * orig_image.height;
-			}
-
-			if (drawHeight > imgCanvas.height) {
-				drawHeight = imgCanvas.height;
-				drawWidth = (drawHeight / orig_image.height) * orig_image.width;
-			}
-
-			imgCtx.drawImage(orig_image, 0, 0, drawWidth, drawHeight);
-
-			// update mask
-			maskCanvas.width = drawWidth;
-			maskCanvas.height = drawHeight;
-			maskCanvas.style.top = imgCanvas.offsetTop + "px";
-			maskCanvas.style.left = imgCanvas.offsetLeft + "px";
-			backupCtx.drawImage(maskCanvas, 0, 0, maskCanvas.width, maskCanvas.height, 0, 0, backupCanvas.width, backupCanvas.height);
-			maskCtx.drawImage(backupCanvas, 0, 0, backupCanvas.width, backupCanvas.height, 0, 0, maskCanvas.width, maskCanvas.height);
-		});
-
 		const filepath = ComfyApp.clipspace.images;
 
-		const touched_image = new Image();
-
-		touched_image.onload = function() {
-			backupCanvas.width = touched_image.width;
-			backupCanvas.height = touched_image.height;
-
-			prepareRGB(touched_image, backupCanvas, backupCtx);
-		};
-
 		const alpha_url = new URL(ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src)
 		alpha_url.searchParams.delete('channel');
 		alpha_url.searchParams.delete('preview');
 		alpha_url.searchParams.set('channel', 'a');
-		touched_image.src = alpha_url;
+		let mask_image = await loadImage(alpha_url);
 
 		// original image load
-		orig_image.onload = function() {
-			window.dispatchEvent(new Event('resize'));
-		};
-
 		const rgb_url = new URL(ComfyApp.clipspace.imgs[ComfyApp.clipspace['selectedIndex']].src);
 		rgb_url.searchParams.delete('channel');
 		rgb_url.searchParams.set('channel', 'rgb');
-		orig_image.src = rgb_url;
+		this.image = new Image();
-		this.image = orig_image;
+		this.image.onload = function() {
+			maskCanvas.width = self.image.width;
+			maskCanvas.height = self.image.height;
+
+			self.invalidateCanvas(self.image, mask_image);
+			self.initializeCanvasPanZoom();
+		};
+		this.image.src = rgb_url;
 	}
 
+	initializeCanvasPanZoom() {
+		// set initialize
+		let drawWidth = this.image.width;
+		let drawHeight = this.image.height;
+
+		let width = this.element.clientWidth;
+		let height = this.element.clientHeight;
+
+		if (this.image.width > width) {
+			drawWidth = width;
+			drawHeight = (drawWidth / this.image.width) * this.image.height;
+		}
+
+		if (drawHeight > height) {
+			drawHeight = height;
+			drawWidth = (drawHeight / this.image.height) * this.image.width;
+		}
+
+		this.zoom_ratio = drawWidth/this.image.width;
+
+		const canvasX = (width - drawWidth) / 2;
+		const canvasY = (height - drawHeight) / 2;
+		this.pan_x = canvasX;
+		this.pan_y = canvasY;
+
+		this.invalidatePanZoom();
+	}
+
+
+	invalidatePanZoom() {
+		let raw_width = this.image.width * this.zoom_ratio;
+		let raw_height = this.image.height * this.zoom_ratio;
+
+		if(this.pan_x + raw_width < 10) {
+			this.pan_x = 10 - raw_width;
+		}
+
+		if(this.pan_y + raw_height < 10) {
+			this.pan_y = 10 - raw_height;
+		}
+
+		let width = `${raw_width}px`;
+		let height = `${raw_height}px`;
+
+		let left = `${this.pan_x}px`;
+		let top = `${this.pan_y}px`;
+
+		this.maskCanvas.style.width = width;
+		this.maskCanvas.style.height = height;
+		this.maskCanvas.style.left = left;
+		this.maskCanvas.style.top = top;
+
+		this.imgCanvas.style.width = width;
+		this.imgCanvas.style.height = height;
+		this.imgCanvas.style.left = left;
+		this.imgCanvas.style.top = top;
+	}
+
+
 	setEventHandler(maskCanvas) {
+		const self = this;
+
+		if(!this.handler_registered) {
 			maskCanvas.addEventListener("contextmenu", (event) => {
 				event.preventDefault();
 			});
 
-		const self = this;
+			this.element.addEventListener('wheel', (event) => this.handleWheelEvent(self,event));
-		maskCanvas.addEventListener('wheel', (event) => this.handleWheelEvent(self,event));
+			this.element.addEventListener('pointermove', (event) => this.pointMoveEvent(self,event));
+			this.element.addEventListener('touchmove', (event) => this.pointMoveEvent(self,event));
+
+			this.element.addEventListener('dragstart', (event) => {
+				if(event.ctrlKey) {
+					event.preventDefault();
+				}
+			});
+
 			maskCanvas.addEventListener('pointerdown', (event) => this.handlePointerDown(self,event));
-		document.addEventListener('pointerup', MaskEditorDialog.handlePointerUp);
 			maskCanvas.addEventListener('pointermove', (event) => this.draw_move(self,event));
 			maskCanvas.addEventListener('touchmove', (event) => this.draw_move(self,event));
 			maskCanvas.addEventListener('pointerover', (event) => { this.brush.style.display = "block"; });
 			maskCanvas.addEventListener('pointerleave', (event) => { this.brush.style.display = "none"; });
-		document.addEventListener('keydown', MaskEditorDialog.handleKeyDown);
+
+			document.addEventListener('pointerup', MaskEditorDialog.handlePointerUp);
+
+			this.handler_registered = true;
+		}
 	}
 
 	brush_size = 10;
@@ -378,8 +455,10 @@ class MaskEditorDialog extends ComfyDialog {
 		const self = MaskEditorDialog.instance;
 		if (event.key === ']') {
 			self.brush_size = Math.min(self.brush_size+2, 100);
+			self.brush_slider_input.value = self.brush_size;
 		} else if (event.key === '[') {
 			self.brush_size = Math.max(self.brush_size-2, 1);
+			self.brush_slider_input.value = self.brush_size;
 		} else if(event.key === 'Enter') {
 			self.save();
 		}
@@ -389,6 +468,10 @@ class MaskEditorDialog extends ComfyDialog {
 
 	static handlePointerUp(event) {
 		event.preventDefault();
+
+		this.mousedown_x = null;
+		this.mousedown_y = null;
+
 		MaskEditorDialog.instance.drawing_mode = false;
 	}
 
@@ -398,24 +481,70 @@ class MaskEditorDialog extends ComfyDialog {
 		var centerX = self.cursorX;
 		var centerY = self.cursorY;
 
-		brush.style.width = self.brush_size * 2 + "px";
+		brush.style.width = self.brush_size * 2 * this.zoom_ratio + "px";
-		brush.style.height = self.brush_size * 2 + "px";
+		brush.style.height = self.brush_size * 2 * this.zoom_ratio + "px";
-		brush.style.left = (centerX - self.brush_size) + "px";
+		brush.style.left = (centerX - self.brush_size * this.zoom_ratio) + "px";
-		brush.style.top = (centerY - self.brush_size) + "px";
+		brush.style.top = (centerY - self.brush_size * this.zoom_ratio) + "px";
 	}
 
 	handleWheelEvent(self, event) {
-		if(event.deltaY < 0)
+		event.preventDefault();
-			self.brush_size = Math.min(self.brush_size+2, 100);
-		else
-			self.brush_size = Math.max(self.brush_size-2, 1);
 
-		self.brush_slider_input.value = self.brush_size;
+		if(event.ctrlKey) {
+			// zoom canvas
+			if(event.deltaY < 0) {
+				this.zoom_ratio = Math.min(10.0, this.zoom_ratio+0.2);
+			}
+			else {
+				this.zoom_ratio = Math.max(0.2, this.zoom_ratio-0.2);
+			}
+
+			this.invalidatePanZoom();
+		}
+		else {
+			// adjust brush size
+			if(event.deltaY < 0)
+				this.brush_size = Math.min(this.brush_size+2, 100);
+			else
+				this.brush_size = Math.max(this.brush_size-2, 1);
+
+			this.brush_slider_input.value = this.brush_size;
+
+			this.updateBrushPreview(this);
+		}
+	}
+
+	pointMoveEvent(self, event) {
+		this.cursorX = event.pageX;
+		this.cursorY = event.pageY;
 
 		self.updateBrushPreview(self);
+
+		if(event.ctrlKey) {
+			event.preventDefault();
+			self.pan_move(self, event);
+		}
+	}
+
+	pan_move(self, event) {
+		if(event.buttons == 1) {
+			if(this.mousedown_x) {
+				let deltaX = this.mousedown_x - event.clientX;
+				let deltaY = this.mousedown_y - event.clientY;
+
+				self.pan_x = this.mousedown_pan_x - deltaX;
+				self.pan_y = this.mousedown_pan_y - deltaY;
+
+				self.invalidatePanZoom();
+			}
+		}
 	}
 
 	draw_move(self, event) {
+		if(event.ctrlKey) {
+			return;
+		}
+
 		event.preventDefault();
 
 		this.cursorX = event.pageX;
@@ -439,6 +568,9 @@ class MaskEditorDialog extends ComfyDialog {
 				y = event.targetTouches[0].clientY - maskRect.top;
 			}
 
+			x /= self.zoom_ratio;
+			y /= self.zoom_ratio;
+
 			var brush_size = this.brush_size;
 			if(event instanceof PointerEvent && event.pointerType == 'pen') {
 				brush_size *= event.pressure;
@@ -489,8 +621,8 @@ class MaskEditorDialog extends ComfyDialog {
 		}
 		else if(event.buttons == 2 || event.buttons == 5 || event.buttons == 32) {
 			const maskRect = self.maskCanvas.getBoundingClientRect();
-			const x = event.offsetX || event.targetTouches[0].clientX - maskRect.left;
+			const x = (event.offsetX || event.targetTouches[0].clientX - maskRect.left) / self.zoom_ratio;
-			const y = event.offsetY || event.targetTouches[0].clientY - maskRect.top;
+			const y = (event.offsetY || event.targetTouches[0].clientY - maskRect.top) / self.zoom_ratio;
 
 			var brush_size = this.brush_size;
 			if(event instanceof PointerEvent && event.pointerType == 'pen') {
@@ -540,6 +672,17 @@ class MaskEditorDialog extends ComfyDialog {
 	}
 
 	handlePointerDown(self, event) {
+		if(event.ctrlKey) {
+			if (event.buttons == 1) {
+				this.mousedown_x = event.clientX;
+				this.mousedown_y = event.clientY;
+
+				this.mousedown_pan_x = this.pan_x;
+				this.mousedown_pan_y = this.pan_y;
+			}
+			return;
+		}
+
 		var brush_size = this.brush_size;
 		if(event instanceof PointerEvent && event.pointerType == 'pen') {
 			brush_size *= event.pressure;
@@ -551,8 +694,8 @@ class MaskEditorDialog extends ComfyDialog {
 
 			event.preventDefault();
 			const maskRect = self.maskCanvas.getBoundingClientRect();
-			const x = event.offsetX || event.targetTouches[0].clientX - maskRect.left;
+			const x = (event.offsetX || event.targetTouches[0].clientX - maskRect.left) / self.zoom_ratio;
-			const y = event.offsetY || event.targetTouches[0].clientY - maskRect.top;
+			const y = (event.offsetY || event.targetTouches[0].clientY - maskRect.top) / self.zoom_ratio;
 
 			self.maskCtx.beginPath();
 			if (event.button == 0) {
@@ -570,15 +713,18 @@ class MaskEditorDialog extends ComfyDialog {
 	}
 
 	async save() {
-		const backupCtx = this.backupCanvas.getContext('2d', {willReadFrequently:true});
+		const backupCanvas = document.createElement('canvas');
+		const backupCtx = backupCanvas.getContext('2d', {willReadFrequently:true});
+		backupCanvas.width = this.image.width;
+		backupCanvas.height = this.image.height;
 
-		backupCtx.clearRect(0,0,this.backupCanvas.width,this.backupCanvas.height);
+		backupCtx.clearRect(0,0, backupCanvas.width, backupCanvas.height);
 		backupCtx.drawImage(this.maskCanvas,
 			0, 0, this.maskCanvas.width, this.maskCanvas.height,
-			0, 0, this.backupCanvas.width, this.backupCanvas.height);
+			0, 0, backupCanvas.width, backupCanvas.height);
 
 		// paste mask data into alpha channel
-		const backupData = backupCtx.getImageData(0, 0, this.backupCanvas.width, this.backupCanvas.height);
+		const backupData = backupCtx.getImageData(0, 0, backupCanvas.width, backupCanvas.height);
 
 		// refine mask image
 		for (let i = 0; i < backupData.data.length; i += 4) {
@@ -615,7 +761,7 @@ class MaskEditorDialog extends ComfyDialog {
 				ComfyApp.clipspace.widgets[index].value = item;
 		}
 
-		const dataURL = this.backupCanvas.toDataURL();
+		const dataURL = backupCanvas.toDataURL();
 		const blob = dataURLToBlob(dataURL);
 
 		let original_url = new URL(this.image.src);