🤖 Add image pasting support to chat input (#151)

Enables pasting images directly into the chat input box for multimodal
AI conversations.

## Changes

**Frontend**
- New `ImageAttachments` component displays pasted images as thumbnails
with remove buttons (80x80px in input, 300x300px max in messages)
- `ChatInput` handles paste events to extract images from clipboard
- Images converted to base64 data URLs for preview and transmission
- Updated send logic to allow sending messages with only images (no text
required)
- `UserMessage` component now renders inline images in chat history

**Backend**
- Extended `CmuxMessage` type with `CmuxImagePart` (image data +
mimeType)
- Updated IPC `sendMessage` to accept optional `imageParts` array
- Modified message construction in `ipcMain` to include image parts
- `StreamingMessageAggregator` extracts image parts when converting to
`DisplayedMessage`
- Images flow through AI SDK's `convertToModelMessages` to provider APIs

**Type Updates**
- `DisplayedMessage` type for user messages now includes optional
`imageParts` field
- Image parts properly preserved through entire message pipeline (input
→ IPC → history → aggregator → UI)

## Implementation Notes

The AI SDK automatically converts image parts to the appropriate format
for each provider (Anthropic, OpenAI, Google, etc.), so no
provider-specific handling is needed.

Image attachments are:
- Cleared on successful send or when explicitly removed by user
- Displayed as thumbnails in input area before sending
- Rendered inline in message history after sending
- Included in chat history for context in subsequent messages

## Testing

Manually tested:
1. Copy image to clipboard
2. Paste into chat input (Cmd+V / Ctrl+V)
3. Verify thumbnail appears with remove button
4. Send message and confirm image renders in chat
5. Verify AI can see and respond to image content

_Generated with `cmux`_

Author: kylecarbs

src/components/ChatInput.tsx

@@ -23,6 +23,7 @@ import { matchesKeybind, formatKeybind, KEYBINDS, isEditableElement } from "@/ut
 import { ModelSelector, type ModelSelectorRef } from "./ModelSelector";
 import { useModelLRU } from "@/hooks/useModelLRU";
 import { VimTextArea } from "./VimTextArea";
+import { ImageAttachments, type ImageAttachment } from "./ImageAttachments";
 
 import type { ThinkingLevel } from "@/types/thinking";
 
@@ -293,6 +294,7 @@ export const ChatInput: React.FC<ChatInputProps> = ({
   const [commandSuggestions, setCommandSuggestions] = useState<SlashSuggestion[]>([]);
   const [providerNames, setProviderNames] = useState<string[]>([]);
   const [toast, setToast] = useState<Toast | null>(null);
+  const [imageAttachments, setImageAttachments] = useState<ImageAttachment[]>([]);
   const handleToastDismiss = useCallback(() => {
     setToast(null);
   }, []);
@@ -450,6 +452,42 @@ export const ChatInput: React.FC<ChatInputProps> = ({
       window.removeEventListener(CUSTOM_EVENTS.THINKING_LEVEL_TOAST, handler as EventListener);
   }, [workspaceId, setToast]);
 
+  // Handle paste events to extract images
+  const handlePaste = useCallback((e: React.ClipboardEvent<HTMLTextAreaElement>) => {
+    const items = e.clipboardData?.items;
+    if (!items) return;
+
+    // Look for image items in clipboard
+    for (const item of Array.from(items)) {
+      if (!item?.type.startsWith("image/")) continue;
+
+      e.preventDefault(); // Prevent default paste behavior for images
+
+      const file = item.getAsFile();
+      if (!file) continue;
+
+      // Convert to base64 data URL
+      const reader = new FileReader();
+      reader.onload = (event) => {
+        const dataUrl = event.target?.result as string;
+        if (dataUrl) {
+          const attachment: ImageAttachment = {
+            id: `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
+            dataUrl,
+            mimeType: file.type,
+          };
+          setImageAttachments((prev) => [...prev, attachment]);
+        }
+      };
+      reader.readAsDataURL(file);
+    }
+  }, []);
+
+  // Handle removing an image attachment
+  const handleRemoveImage = useCallback((id: string) => {
+    setImageAttachments((prev) => prev.filter((img) => img.id !== id));
+  }, []);
+
   // Handle command selection
   const handleCommandSelect = useCallback(
     (suggestion: SlashSuggestion) => {
@@ -461,7 +499,9 @@ export const ChatInput: React.FC<ChatInputProps> = ({
   );
 
   const handleSend = async () => {
-    if (!input.trim() || disabled || isSending || isCompacting) return;
+    // Allow sending if there's text or images
+    if ((!input.trim() && imageAttachments.length === 0) || disabled || isSending || isCompacting)
+      return;
 
     const messageText = input.trim();
 
@@ -608,9 +648,16 @@ export const ChatInput: React.FC<ChatInputProps> = ({
       setIsSending(true);
 
       try {
+        // Prepare image parts if any
+        const imageParts = imageAttachments.map((img) => ({
+          image: img.dataUrl,
+          mimeType: img.mimeType,
+        }));
+
         const result = await window.api.workspace.sendMessage(workspaceId, messageText, {
           ...sendMessageOptions,
           editMessageId: editingMessage?.id,
+          imageParts: imageParts.length > 0 ? imageParts : undefined,
         });
 
         if (!result.success) {
@@ -621,8 +668,9 @@ export const ChatInput: React.FC<ChatInputProps> = ({
           // Restore input on error so user can try again
           setInput(messageText);
         } else {
-          // Success - clear input
+          // Success - clear input and images
           setInput("");
+          setImageAttachments([]);
           // Reset textarea height
           if (inputRef.current) {
             inputRef.current.style.height = "36px";
@@ -743,6 +791,7 @@ export const ChatInput: React.FC<ChatInputProps> = ({
           mode={mode}
           onChange={setInput}
           onKeyDown={handleKeyDown}
+          onPaste={handlePaste}
           suppressKeys={showCommandSuggestions ? COMMAND_SUGGESTION_KEYS : undefined}
           placeholder={placeholder}
           disabled={disabled || isSending || isCompacting}
@@ -754,6 +803,7 @@ export const ChatInput: React.FC<ChatInputProps> = ({
           aria-expanded={showCommandSuggestions && commandSuggestions.length > 0}
         />
       </InputControls>
+      <ImageAttachments images={imageAttachments} onRemove={handleRemoveImage} />
       <ModeToggles data-component="ChatModeToggles">
         {editingMessage && (
           <EditingIndicator>

src/components/ImageAttachments.tsx

@@ -0,0 +1,76 @@
+import React from "react";
+import styled from "@emotion/styled";
+
+const AttachmentsContainer = styled.div`
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+  padding: 8px 0;
+`;
+
+const ImagePreview = styled.div`
+  position: relative;
+  width: 80px;
+  height: 80px;
+  border-radius: 4px;
+  overflow: hidden;
+  border: 1px solid #3e3e42;
+  background: #1e1e1e;
+`;
+
+const PreviewImage = styled.img`
+  width: 100%;
+  height: 100%;
+  object-fit: cover;
+`;
+
+const RemoveButton = styled.button`
+  position: absolute;
+  top: 4px;
+  right: 4px;
+  width: 20px;
+  height: 20px;
+  border-radius: 50%;
+  background: rgba(0, 0, 0, 0.7);
+  color: white;
+  border: none;
+  cursor: pointer;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  font-size: 14px;
+  line-height: 1;
+  padding: 0;
+
+  &:hover {
+    background: rgba(0, 0, 0, 0.9);
+  }
+`;
+
+export interface ImageAttachment {
+  id: string;
+  dataUrl: string;
+  mimeType: string;
+}
+
+interface ImageAttachmentsProps {
+  images: ImageAttachment[];
+  onRemove: (id: string) => void;
+}
+
+export const ImageAttachments: React.FC<ImageAttachmentsProps> = ({ images, onRemove }) => {
+  if (images.length === 0) return null;
+
+  return (
+    <AttachmentsContainer>
+      {images.map((image) => (
+        <ImagePreview key={image.id}>
+          <PreviewImage src={image.dataUrl} alt="Attached image" />
+          <RemoveButton onClick={() => onRemove(image.id)} title="Remove image">
+            ×
+          </RemoveButton>
+        </ImagePreview>
+      ))}
+    </AttachmentsContainer>
+  );
+};

src/components/Messages/UserMessage.tsx

@@ -16,6 +16,20 @@ const FormattedContent = styled.pre`
   opacity: 0.9;
 `;
 
+const ImageContainer = styled.div`
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+  margin-top: 8px;
+`;
+
+const MessageImage = styled.img`
+  max-width: 300px;
+  max-height: 300px;
+  border-radius: 4px;
+  border: 1px solid #3e3e42;
+`;
+
 interface UserMessageProps {
   message: DisplayedMessage & { type: "user" };
   className?: string;
@@ -91,7 +105,14 @@ export const UserMessage: React.FC<UserMessageProps> = ({ message, className, on
       buttons={buttons}
       className={className}
     >
-      <FormattedContent>{content}</FormattedContent>
+      {content && <FormattedContent>{content}</FormattedContent>}
+      {message.imageParts && message.imageParts.length > 0 && (
+        <ImageContainer>
+          {message.imageParts.map((img, idx) => (
+            <MessageImage key={idx} src={img.image} alt={`Attachment ${idx + 1}`} />
+          ))}
+        </ImageContainer>
+      )}
     </MessageWindow>
   );
 };

src/services/ipcMain.ts

@@ -492,7 +492,12 @@ export class IpcMain {
 
     ipcMain.handle(
       IPC_CHANNELS.WORKSPACE_SEND_MESSAGE,
-      async (_event, workspaceId: string, message: string, options?: SendMessageOptions) => {
+      async (
+        _event,
+        workspaceId: string,
+        message: string,
+        options?: SendMessageOptions & { imageParts?: Array<{ image: string; mimeType: string }> }
+      ) => {
         const {
           editMessageId,
           thinkingLevel,
@@ -501,6 +506,7 @@ export class IpcMain {
           additionalSystemInstructions,
           maxOutputTokens,
           providerOptions,
+          imageParts,
         } = options ?? {};
         log.debug("sendMessage handler: Received", {
           workspaceId,
@@ -514,9 +520,9 @@ export class IpcMain {
           providerOptions,
         });
         try {
-          // Early exit: empty message = either interrupt (if streaming) or invalid input
+          // Early exit: empty message and no images = either interrupt (if streaming) or invalid input
           // This prevents race conditions where empty messages arrive after streaming stops
-          if (!message.trim()) {
+          if (!message.trim() && (!imageParts || imageParts.length === 0)) {
             // If streaming, this is an interrupt request (from Esc key)
             if (this.aiService.isStreaming(workspaceId)) {
               log.debug("sendMessage handler: Empty message during streaming, interrupting");
@@ -553,13 +559,30 @@ export class IpcMain {
             // replacement automatically when the new message arrives with the same historySequence
           }
 
-          // Create user message
+          // Create user message with text and optional image parts
           const messageId = `user-${Date.now()}-${Math.random().toString(36).substring(2, 11)}`;
-          const userMessage = createCmuxMessage(messageId, "user", message, {
-            // historySequence will be assigned by historyService.appendToHistory()
-            timestamp: Date.now(),
-            toolPolicy, // Store for historical record and compaction detection
-          });
+          const additionalParts = imageParts?.map((img) => ({
+            type: "image" as const,
+            image: img.image,
+            mimeType: img.mimeType,
+          }));
+          if (additionalParts && additionalParts.length > 0) {
+            log.debug("sendMessage: Creating message with images", {
+              imageCount: additionalParts.length,
+              mimeTypes: additionalParts.map((p) => p.mimeType),
+            });
+          }
+          const userMessage = createCmuxMessage(
+            messageId,
+            "user",
+            message,
+            {
+              // historySequence will be assigned by historyService.appendToHistory()
+              timestamp: Date.now(),
+              toolPolicy, // Store for historical record and compaction detection
+            },
+            additionalParts
+          );
 
           // Append user message to history
           const appendResult = await this.historyService.appendToHistory(workspaceId, userMessage);

src/types/ipc.ts

@@ -177,7 +177,7 @@ export interface IPCApi {
     sendMessage(
       workspaceId: string,
       message: string,
-      options?: SendMessageOptions
+      options?: SendMessageOptions & { imageParts?: Array<{ image: string; mimeType: string }> }
     ): Promise<Result<void, SendMessageError>>;
     resumeStream(
       workspaceId: string,

src/types/message.ts

@@ -43,10 +43,17 @@ export interface CmuxReasoningPart {
   text: string;
 }
 
+// Image part type for multimodal messages
+export interface CmuxImagePart {
+  type: "image";
+  image: string | Uint8Array | ArrayBuffer | URL; // base64 string or binary data or URL
+  mimeType?: string; // e.g., "image/png", "image/jpeg"
+}
+
 // CmuxMessage extends UIMessage with our metadata and custom parts
-// Supports text, reasoning, and tool parts (including interrupted tool calls)
+// Supports text, reasoning, image, and tool parts (including interrupted tool calls)
 export type CmuxMessage = Omit<UIMessage<CmuxMetadata, never, never>, "parts"> & {
-  parts: Array<CmuxTextPart | CmuxReasoningPart | CmuxToolPart>;
+  parts: Array<CmuxTextPart | CmuxReasoningPart | CmuxImagePart | CmuxToolPart>;
 };
 
 // DisplayedMessage represents a single UI message block
@@ -57,6 +64,7 @@ export type DisplayedMessage =
       id: string; // Display ID for UI/React keys
       historyId: string; // Original CmuxMessage ID for history operations
       content: string;
+      imageParts?: Array<{ image: string; mimeType?: string }>; // Optional image attachments
       historySequence: number; // Global ordering across all messages
       timestamp?: number;
     }

src/utils/messages/StreamingMessageAggregator.ts

@@ -415,17 +415,25 @@ export class StreamingMessageAggregator {
       const historySequence = message.metadata?.historySequence ?? 0;
 
       if (message.role === "user") {
-        // User messages: combine all text parts into single block
+        // User messages: combine all text parts into single block, extract images
         const content = message.parts
           .filter((p) => p.type === "text")
           .map((p) => p.text)
           .join("");
 
+        const imageParts = message.parts
+          .filter((p) => p.type === "image")
+          .map((p) => ({
+            image: typeof p.image === "string" ? p.image : "",
+            mimeType: p.mimeType,
+          }));
+
         displayedMessages.push({
           type: "user",
           id: message.id,
           historyId: message.id,
           content,
+          imageParts: imageParts.length > 0 ? imageParts : undefined,
           historySequence,
           timestamp: baseTimestamp,
         });

src/utils/messages/modelMessageTransform.ts

@@ -413,10 +413,18 @@ function mergeConsecutiveUserMessages(messages: ModelMessage[]): ModelMessage[]
       // Merge with newline prefix
       const mergedText = prevText + "\n" + currentText;
 
-      // Update the previous message
+      // Collect image parts from both messages
+      const prevImageParts = Array.isArray(prevMsg.content)
+        ? prevMsg.content.filter((c) => c.type === "image")
+        : [];
+      const currentImageParts = Array.isArray(msg.content)
+        ? msg.content.filter((c) => c.type === "image")
+        : [];
+
+      // Update the previous message with merged text and all image parts
       merged[merged.length - 1] = {
         role: "user",
-        content: [{ type: "text", text: mergedText }],
+        content: [{ type: "text", text: mergedText }, ...prevImageParts, ...currentImageParts],
       };
     } else {
       // Not consecutive user message, add as-is