diff --git a/packages/excalidraw/components/LibraryMenu.tsx b/packages/excalidraw/components/LibraryMenu.tsx
index 0aa6071aa0..9a4f29f179 100644
--- a/packages/excalidraw/components/LibraryMenu.tsx
+++ b/packages/excalidraw/components/LibraryMenu.tsx
@@ -281,19 +281,29 @@ export const LibraryMenu = memo(() => {
           if (target.closest(`.${CLASSES.SIDEBAR}`)) {
             // stop propagation so that we don't prevent it downstream
             // (default browser behavior is to clear search input on ESC)
-            event.stopPropagation();
             if (selectedItems.length > 0) {
+              event.stopPropagation();
               setSelectedItems([]);
             } else if (
               isWritableElement(target) &&
               target instanceof HTMLInputElement &&
               !target.value
             ) {
+              event.stopPropagation();
               // if search input empty -> close library
               // (maybe not a good idea?)
               setAppState({ openSidebar: null });
               app.focusContainer();
             }
+          } else if (selectedItems.length > 0) {
+            const { x, y } = app.lastViewportPosition;
+            const elementUnderCursor = document.elementFromPoint(x, y);
+            // also deselect elements if sidebar doesn't have focus but the
+            // cursor is over it
+            if (elementUnderCursor?.closest(`.${CLASSES.SIDEBAR}`)) {
+              event.stopPropagation();
+              setSelectedItems([]);
+            }
           }
         }
       },
diff --git a/packages/excalidraw/components/LibraryMenuItems.tsx b/packages/excalidraw/components/LibraryMenuItems.tsx
index 2d111b7f7b..c64351b1b3 100644
--- a/packages/excalidraw/components/LibraryMenuItems.tsx
+++ b/packages/excalidraw/components/LibraryMenuItems.tsx
@@ -138,10 +138,13 @@ export default function LibraryMenuItems({
           }
 
           const selectedItemsMap = arrayToMap(selectedItems);
+          // Support both top-down and bottom-up selection by using min/max
+          const minRange = Math.min(rangeStart, rangeEnd);
+          const maxRange = Math.max(rangeStart, rangeEnd);
           const nextSelectedIds = orderedItems.reduce(
             (acc: LibraryItem["id"][], item, idx) => {
               if (
-                (idx >= rangeStart && idx <= rangeEnd) ||
+                (idx >= minRange && idx <= maxRange) ||
                 selectedItemsMap.has(item.id)
               ) {
                 acc.push(item.id);
@@ -169,6 +172,14 @@ export default function LibraryMenuItems({
     ],
   );
 
+  useEffect(() => {
+    // if selection is removed (e.g. via esc), reset last selected item
+    // so that subsequent shift+clicks don't select a large range
+    if (!selectedItems.length) {
+      setLastSelectedItem(null);
+    }
+  }, [selectedItems]);
+
   const getInsertedElements = useCallback(
     (id: string) => {
       let targetElements;