feat: [cont.] support inserting multiple images (#9875)

* feat: support inserting multiple images

* Initial

* handleAppOnDrop, onImageToolbarButtonClick, pasteFromClipboard

* Initial get history working

* insertMultipleImages -> insertImages

* Bug fixes, improvements

* Remove redundant branch

* Refactor addElementsFromMixedContentPaste

* History, drag & drop bug fixes

* Update snapshots

* Remove redundant try-catch

* Refactor pasteFromClipboard

* Plain paste check in mermaid paste

* Move comment

* processClipboardData -> insertClipboardContent

* Redundant variable

* Redundant variable

* Refactor insertImages

* createImagePlaceholder -> newImagePlaceholder

* Get rid of unneeded NEVER schedule, filter out failed images

* Trigger CI

* Position placeholders before initializing

* Don't mutate scene with positionElementsOnGrid, captureUpdate: CaptureUpdateAction.IMMEDIATELY

* Comment

* Move positionOnGrid out of file

* Rename file

* Get rid of generic

* Initial tests

* More asserts, test paste

* Test image tool

* De-duplicate

* Stricter assert, move rest of logic outside of waitFor

* Modify history tests

* De-duplicate update snapshots

* Trigger CI

* Fix package build

* Make setupImageTest more explicit

* Re-introduce generic to use latest placeholder versions

* newElementWith instead of mutateElement to delete failed placeholder

* Insert failed images separately with CaptureUpdateAction.NEVER

* Refactor

* Don't re-order elements

* WIP

* Get rid of 'never' for failed

* refactor type check

* align max file size constant

* make grid padding scale to zoom

---------

Co-authored-by: dwelle <5153846+dwelle@users.noreply.github.com>
This commit is contained in:
Omar Brikaa
2025-09-01 18:31:24 +03:00
committed by GitHub
parent ae89608985
commit 3bdaafe4b5
15 changed files with 895 additions and 502 deletions
+281 -306
View File
@@ -237,6 +237,7 @@ import {
isSimpleArrow,
StoreDelta,
type ApplyToOptions,
positionElementsOnGrid,
} from "@excalidraw/element";
import type { LocalPoint, Radians } from "@excalidraw/math";
@@ -345,7 +346,7 @@ import {
generateIdFromFile,
getDataURL,
getDataURL_sync,
getFileFromEvent,
getFilesFromEvent,
ImageURLToFile,
isImageFileHandle,
isSupportedImageFile,
@@ -432,7 +433,7 @@ import type {
ScrollBars,
} from "../scene/types";
import type { PastedMixedContent } from "../clipboard";
import type { ClipboardData, PastedMixedContent } from "../clipboard";
import type { ExportedElements } from "../data";
import type { ContextMenuItems } from "./ContextMenu";
import type { FileSystemHandle } from "../data/filesystem";
@@ -3066,7 +3067,168 @@ class App extends React.Component<AppProps, AppState> {
}
};
// TODO: this is so spaghetti, we should refactor it and cover it with tests
// TODO: Cover with tests
private async insertClipboardContent(
data: ClipboardData,
filesData: Awaited<ReturnType<typeof getFilesFromEvent>>,
isPlainPaste: boolean,
) {
const { x: sceneX, y: sceneY } = viewportCoordsToSceneCoords(
{
clientX: this.lastViewportPosition.x,
clientY: this.lastViewportPosition.y,
},
this.state,
);
// ------------------- Error -------------------
if (data.errorMessage) {
this.setState({ errorMessage: data.errorMessage });
return;
}
// ------------------- Mixed content with no files -------------------
if (filesData.length === 0 && !isPlainPaste && data.mixedContent) {
await this.addElementsFromMixedContentPaste(data.mixedContent, {
isPlainPaste,
sceneX,
sceneY,
});
return;
}
// ------------------- Spreadsheet -------------------
if (data.spreadsheet && !isPlainPaste) {
this.setState({
pasteDialog: {
data: data.spreadsheet,
shown: true,
},
});
return;
}
// ------------------- Images or SVG code -------------------
const imageFiles = filesData
.map((data) => data.file)
.filter((file): file is File => isSupportedImageFile(file));
if (imageFiles.length === 0 && data.text && !isPlainPaste) {
const trimmedText = data.text.trim();
if (trimmedText.startsWith("<svg") && trimmedText.endsWith("</svg>")) {
// ignore SVG validation/normalization which will be done during image
// initialization
imageFiles.push(SVGStringToFile(trimmedText));
}
}
if (imageFiles.length > 0) {
if (this.isToolSupported("image")) {
await this.insertImages(imageFiles, sceneX, sceneY);
} else {
this.setState({ errorMessage: t("errors.imageToolNotSupported") });
}
return;
}
// ------------------- Elements -------------------
if (data.elements) {
const elements = (
data.programmaticAPI
? convertToExcalidrawElements(
data.elements as ExcalidrawElementSkeleton[],
)
: data.elements
) as readonly ExcalidrawElement[];
// TODO: remove formatting from elements if isPlainPaste
this.addElementsFromPasteOrLibrary({
elements,
files: data.files || null,
position: this.isMobileOrTablet() ? "center" : "cursor",
retainSeed: isPlainPaste,
});
return;
}
// ------------------- Only textual stuff remaining -------------------
if (!data.text) {
return;
}
// ------------------- Successful Mermaid -------------------
if (!isPlainPaste && isMaybeMermaidDefinition(data.text)) {
const api = await import("@excalidraw/mermaid-to-excalidraw");
try {
const { elements: skeletonElements, files } =
await api.parseMermaidToExcalidraw(data.text);
const elements = convertToExcalidrawElements(skeletonElements, {
regenerateIds: true,
});
this.addElementsFromPasteOrLibrary({
elements,
files,
position: this.isMobileOrTablet() ? "center" : "cursor",
});
return;
} catch (err: any) {
console.warn(
`parsing pasted text as mermaid definition failed: ${err.message}`,
);
}
}
// ------------------- Pure embeddable URLs -------------------
const nonEmptyLines = normalizeEOL(data.text)
.split(/\n+/)
.map((s) => s.trim())
.filter(Boolean);
const embbeddableUrls = nonEmptyLines
.map((str) => maybeParseEmbedSrc(str))
.filter(
(string) =>
embeddableURLValidator(string, this.props.validateEmbeddable) &&
(/^(http|https):\/\/[^\s/$.?#].[^\s]*$/.test(string) ||
getEmbedLink(string)?.type === "video"),
);
if (
!isPlainPaste &&
embbeddableUrls.length > 0 &&
embbeddableUrls.length === nonEmptyLines.length
) {
const embeddables: NonDeleted<ExcalidrawEmbeddableElement>[] = [];
for (const url of embbeddableUrls) {
const prevEmbeddable: ExcalidrawEmbeddableElement | undefined =
embeddables[embeddables.length - 1];
const embeddable = this.insertEmbeddableElement({
sceneX: prevEmbeddable
? prevEmbeddable.x + prevEmbeddable.width + 20
: sceneX,
sceneY,
link: normalizeLink(url),
});
if (embeddable) {
embeddables.push(embeddable);
}
}
if (embeddables.length) {
this.store.scheduleCapture();
this.setState({
selectedElementIds: Object.fromEntries(
embeddables.map((embeddable) => [embeddable.id, true]),
),
});
}
return;
}
// ------------------- Text -------------------
this.addTextFromPaste(data.text, isPlainPaste);
}
public pasteFromClipboard = withBatchedUpdates(
async (event: ClipboardEvent) => {
const isPlainPaste = !!IS_PLAIN_PASTE;
@@ -3091,47 +3253,11 @@ class App extends React.Component<AppProps, AppState> {
return;
}
const { x: sceneX, y: sceneY } = viewportCoordsToSceneCoords(
{
clientX: this.lastViewportPosition.x,
clientY: this.lastViewportPosition.y,
},
this.state,
);
// must be called in the same frame (thus before any awaits) as the paste
// event else some browsers (FF...) will clear the clipboardData
// (something something security)
let file = event?.clipboardData?.files[0];
const filesData = await getFilesFromEvent(event);
const data = await parseClipboard(event, isPlainPaste);
if (!file && !isPlainPaste) {
if (data.mixedContent) {
return this.addElementsFromMixedContentPaste(data.mixedContent, {
isPlainPaste,
sceneX,
sceneY,
});
} else if (data.text) {
const string = data.text.trim();
if (string.startsWith("<svg") && string.endsWith("</svg>")) {
// ignore SVG validation/normalization which will be done during image
// initialization
file = SVGStringToFile(string);
}
}
}
// prefer spreadsheet data over image file (MS Office/Libre Office)
if (isSupportedImageFile(file) && !data.spreadsheet) {
if (!this.isToolSupported("image")) {
this.setState({ errorMessage: t("errors.imageToolNotSupported") });
return;
}
this.createImageElement({ sceneX, sceneY, imageFile: file });
return;
}
if (this.props.onPaste) {
try {
@@ -3143,105 +3269,7 @@ class App extends React.Component<AppProps, AppState> {
}
}
if (data.errorMessage) {
this.setState({ errorMessage: data.errorMessage });
} else if (data.spreadsheet && !isPlainPaste) {
this.setState({
pasteDialog: {
data: data.spreadsheet,
shown: true,
},
});
} else if (data.elements) {
const elements = (
data.programmaticAPI
? convertToExcalidrawElements(
data.elements as ExcalidrawElementSkeleton[],
)
: data.elements
) as readonly ExcalidrawElement[];
// TODO remove formatting from elements if isPlainPaste
this.addElementsFromPasteOrLibrary({
elements,
files: data.files || null,
position: this.isMobileOrTablet() ? "center" : "cursor",
retainSeed: isPlainPaste,
});
} else if (data.text) {
if (data.text && isMaybeMermaidDefinition(data.text)) {
const api = await import("@excalidraw/mermaid-to-excalidraw");
try {
const { elements: skeletonElements, files } =
await api.parseMermaidToExcalidraw(data.text);
const elements = convertToExcalidrawElements(skeletonElements, {
regenerateIds: true,
});
this.addElementsFromPasteOrLibrary({
elements,
files,
position: this.isMobileOrTablet() ? "center" : "cursor",
});
return;
} catch (err: any) {
console.warn(
`parsing pasted text as mermaid definition failed: ${err.message}`,
);
}
}
const nonEmptyLines = normalizeEOL(data.text)
.split(/\n+/)
.map((s) => s.trim())
.filter(Boolean);
const embbeddableUrls = nonEmptyLines
.map((str) => maybeParseEmbedSrc(str))
.filter((string) => {
return (
embeddableURLValidator(string, this.props.validateEmbeddable) &&
(/^(http|https):\/\/[^\s/$.?#].[^\s]*$/.test(string) ||
getEmbedLink(string)?.type === "video")
);
});
if (
!IS_PLAIN_PASTE &&
embbeddableUrls.length > 0 &&
// if there were non-embeddable text (lines) mixed in with embeddable
// urls, ignore and paste as text
embbeddableUrls.length === nonEmptyLines.length
) {
const embeddables: NonDeleted<ExcalidrawEmbeddableElement>[] = [];
for (const url of embbeddableUrls) {
const prevEmbeddable: ExcalidrawEmbeddableElement | undefined =
embeddables[embeddables.length - 1];
const embeddable = this.insertEmbeddableElement({
sceneX: prevEmbeddable
? prevEmbeddable.x + prevEmbeddable.width + 20
: sceneX,
sceneY,
link: normalizeLink(url),
});
if (embeddable) {
embeddables.push(embeddable);
}
}
if (embeddables.length) {
this.store.scheduleCapture();
this.setState({
selectedElementIds: Object.fromEntries(
embeddables.map((embeddable) => [embeddable.id, true]),
),
});
}
return;
}
this.addTextFromPaste(data.text, isPlainPaste);
}
await this.insertClipboardContent(data, filesData, isPlainPaste);
this.setActiveTool({ type: this.defaultSelectionTool }, true);
event?.preventDefault();
},
@@ -3431,45 +3459,11 @@ class App extends React.Component<AppProps, AppState> {
}
}),
);
let y = sceneY;
let firstImageYOffsetDone = false;
const nextSelectedIds: Record<ExcalidrawElement["id"], true> = {};
for (const response of responses) {
if (response.file) {
const initializedImageElement = await this.createImageElement({
sceneX,
sceneY: y,
imageFile: response.file,
});
if (initializedImageElement) {
// vertically center first image in the batch
if (!firstImageYOffsetDone) {
firstImageYOffsetDone = true;
y -= initializedImageElement.height / 2;
}
// hack to reset the `y` coord because we vertically center during
// insertImageElement
this.scene.mutateElement(
initializedImageElement,
{ y },
{ informMutation: false, isDragging: false },
);
y = initializedImageElement.y + initializedImageElement.height + 25;
nextSelectedIds[initializedImageElement.id] = true;
}
}
}
this.setState({
selectedElementIds: makeNextSelectedElementIds(
nextSelectedIds,
this.state,
),
});
const imageFiles = responses
.filter((response): response is { file: File } => !!response.file)
.map((response) => response.file);
await this.insertImages(imageFiles, sceneX, sceneY);
const error = responses.find((response) => !!response.errorMessage);
if (error && error.errorMessage) {
this.setState({ errorMessage: error.errorMessage });
@@ -4806,7 +4800,7 @@ class App extends React.Component<AppProps, AppState> {
this.setState({ suggestedBindings: [] });
}
if (nextActiveTool.type === "image") {
this.onImageAction();
this.onImageToolbarButtonClick();
}
this.setState((prevState) => {
@@ -7842,16 +7836,14 @@ class App extends React.Component<AppProps, AppState> {
return element;
};
private createImageElement = async ({
private newImagePlaceholder = ({
sceneX,
sceneY,
addToFrameUnderCursor = true,
imageFile,
}: {
sceneX: number;
sceneY: number;
addToFrameUnderCursor?: boolean;
imageFile: File;
}) => {
const [gridX, gridY] = getGridPoint(
sceneX,
@@ -7870,7 +7862,7 @@ class App extends React.Component<AppProps, AppState> {
const placeholderSize = 100 / this.state.zoom.value;
const placeholderImageElement = newImageElement({
return newImageElement({
type: "image",
strokeColor: this.state.currentItemStrokeColor,
backgroundColor: this.state.currentItemBackgroundColor,
@@ -7887,13 +7879,6 @@ class App extends React.Component<AppProps, AppState> {
width: placeholderSize,
height: placeholderSize,
});
const initializedImageElement = await this.insertImageElement(
placeholderImageElement,
imageFile,
);
return initializedImageElement;
};
private handleLinearElementOnPointerDown = (
@@ -10215,64 +10200,7 @@ class App extends React.Component<AppProps, AppState> {
);
};
/**
* inserts image into elements array and rerenders
*/
private insertImageElement = async (
placeholderImageElement: ExcalidrawImageElement,
imageFile: File,
) => {
// we should be handling all cases upstream, but in case we forget to handle
// a future case, let's throw here
if (!this.isToolSupported("image")) {
this.setState({ errorMessage: t("errors.imageToolNotSupported") });
return;
}
this.scene.insertElement(placeholderImageElement);
try {
const initializedImageElement = await this.initializeImage(
placeholderImageElement,
imageFile,
);
const nextElements = this.scene
.getElementsIncludingDeleted()
.map((element) => {
if (element.id === initializedImageElement.id) {
return initializedImageElement;
}
return element;
});
this.updateScene({
captureUpdate: CaptureUpdateAction.IMMEDIATELY,
elements: nextElements,
appState: {
selectedElementIds: makeNextSelectedElementIds(
{ [initializedImageElement.id]: true },
this.state,
),
},
});
return initializedImageElement;
} catch (error: any) {
this.store.scheduleAction(CaptureUpdateAction.NEVER);
this.scene.mutateElement(placeholderImageElement, {
isDeleted: true,
});
this.actionManager.executeAction(actionFinalize);
this.setState({
errorMessage: error.message || t("errors.imageInsertError"),
});
return null;
}
};
private onImageAction = async () => {
private onImageToolbarButtonClick = async () => {
try {
const clientX = this.state.width / 2 + this.state.offsetLeft;
const clientY = this.state.height / 2 + this.state.offsetTop;
@@ -10282,24 +10210,15 @@ class App extends React.Component<AppProps, AppState> {
this.state,
);
const imageFile = await fileOpen({
const imageFiles = await fileOpen({
description: "Image",
extensions: Object.keys(
IMAGE_MIME_TYPES,
) as (keyof typeof IMAGE_MIME_TYPES)[],
multiple: true,
});
await this.createImageElement({
sceneX: x,
sceneY: y,
addToFrameUnderCursor: false,
imageFile,
});
// avoid being batched (just in case)
this.setState({}, () => {
this.actionManager.executeAction(actionFinalize);
});
this.insertImages(imageFiles, x, y);
} catch (error: any) {
if (error.name !== "AbortError") {
console.error(error);
@@ -10496,60 +10415,113 @@ class App extends React.Component<AppProps, AppState> {
}
};
private insertImages = async (
imageFiles: File[],
sceneX: number,
sceneY: number,
) => {
const gridPadding = 50 / this.state.zoom.value;
// Create, position, and insert placeholders
const placeholders = positionElementsOnGrid(
imageFiles.map(() => this.newImagePlaceholder({ sceneX, sceneY })),
sceneX,
sceneY,
gridPadding,
);
placeholders.forEach((el) => this.scene.insertElement(el));
// Create, position, insert and select initialized (replacing placeholders)
const initialized = await Promise.all(
placeholders.map(async (placeholder, i) => {
try {
return await this.initializeImage(placeholder, imageFiles[i]);
} catch (error: any) {
this.setState({
errorMessage: error.message || t("errors.imageInsertError"),
});
return newElementWith(placeholder, { isDeleted: true });
}
}),
);
const initializedMap = arrayToMap(initialized);
const positioned = positionElementsOnGrid(
initialized.filter((el) => !el.isDeleted),
sceneX,
sceneY,
gridPadding,
);
const positionedMap = arrayToMap(positioned);
const nextElements = this.scene
.getElementsIncludingDeleted()
.map((el) => positionedMap.get(el.id) ?? initializedMap.get(el.id) ?? el);
this.updateScene({
appState: {
selectedElementIds: makeNextSelectedElementIds(
Object.fromEntries(positioned.map((el) => [el.id, true])),
this.state,
),
},
elements: nextElements,
captureUpdate: CaptureUpdateAction.IMMEDIATELY,
});
this.setState({}, () => {
// actionFinalize after all state values have been updated
this.actionManager.executeAction(actionFinalize);
});
};
private handleAppOnDrop = async (event: React.DragEvent<HTMLDivElement>) => {
// must be retrieved first, in the same frame
const { file, fileHandle } = await getFileFromEvent(event);
const { x: sceneX, y: sceneY } = viewportCoordsToSceneCoords(
event,
this.state,
);
try {
// if image tool not supported, don't show an error here and let it fall
// through so we still support importing scene data from images. If no
// scene data encoded, we'll show an error then
if (isSupportedImageFile(file) && this.isToolSupported("image")) {
// first attempt to decode scene from the image if it's embedded
// ---------------------------------------------------------------------
// must be retrieved first, in the same frame
const filesData = await getFilesFromEvent(event);
if (file?.type === MIME_TYPES.png || file?.type === MIME_TYPES.svg) {
try {
const scene = await loadFromBlob(
file,
this.state,
this.scene.getElementsIncludingDeleted(),
fileHandle,
);
this.syncActionResult({
...scene,
appState: {
...(scene.appState || this.state),
isLoading: false,
},
replaceFiles: true,
captureUpdate: CaptureUpdateAction.IMMEDIATELY,
});
return;
} catch (error: any) {
// Don't throw for image scene daa
if (error.name !== "EncodingError") {
throw new Error(t("alerts.couldNotLoadInvalidFile"));
}
if (filesData.length === 1) {
const { file, fileHandle } = filesData[0];
if (
file &&
(file.type === MIME_TYPES.png || file.type === MIME_TYPES.svg)
) {
try {
const scene = await loadFromBlob(
file,
this.state,
this.scene.getElementsIncludingDeleted(),
fileHandle,
);
this.syncActionResult({
...scene,
appState: {
...(scene.appState || this.state),
isLoading: false,
},
replaceFiles: true,
captureUpdate: CaptureUpdateAction.IMMEDIATELY,
});
return;
} catch (error: any) {
if (error.name !== "EncodingError") {
throw new Error(t("alerts.couldNotLoadInvalidFile"));
}
// if EncodingError, fall through to insert as regular image
}
// if no scene is embedded or we fail for whatever reason, fall back
// to importing as regular image
// ---------------------------------------------------------------------
this.createImageElement({ sceneX, sceneY, imageFile: file });
return;
}
} catch (error: any) {
return this.setState({
isLoading: false,
errorMessage: error.message,
});
}
const imageFiles = filesData
.map((data) => data.file)
.filter((file): file is File => isSupportedImageFile(file));
if (imageFiles.length > 0 && this.isToolSupported("image")) {
return this.insertImages(imageFiles, sceneX, sceneY);
}
const libraryJSON = event.dataTransfer.getData(MIME_TYPES.excalidrawlib);
@@ -10567,9 +10539,12 @@ class App extends React.Component<AppProps, AppState> {
return;
}
if (file) {
// Attempt to parse an excalidraw/excalidrawlib file
await this.loadFileToCanvas(file, fileHandle);
if (filesData.length > 0) {
const { file, fileHandle } = filesData[0];
if (file) {
// Attempt to parse an excalidraw/excalidrawlib file
await this.loadFileToCanvas(file, fileHandle);
}
}
if (event.dataTransfer?.types?.includes("text/plain")) {