Play sound effects for notifications and events (#38)

Co-authored-by: HeavenOSK <heavenosk@gmail.com>

package-lock.json

@@ -32,6 +32,7 @@
         "os-name": "^6.0.0",
         "p-wait-for": "^5.0.2",
         "pdf-parse": "^1.1.1",
+        "play-sound": "^1.1.6",
         "puppeteer-chromium-resolver": "^23.0.0",
         "puppeteer-core": "^23.4.0",
         "serialize-error": "^11.0.3",
@@ -8823,6 +8824,14 @@
         "node": ">=8"
       }
     },
+    "node_modules/find-exec": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/find-exec/-/find-exec-1.0.3.tgz",
+      "integrity": "sha512-gnG38zW90mS8hm5smNcrBnakPEt+cGJoiMkJwCU0IYnEb0H2NQk0NIljhNW+48oniCriFek/PH6QXbwsJo/qug==",
+      "dependencies": {
+        "shell-quote": "^1.8.1"
+      }
+    },
     "node_modules/find-up": {
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz",
@@ -12788,6 +12797,14 @@
         "node": ">=8"
       }
     },
+    "node_modules/play-sound": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/play-sound/-/play-sound-1.1.6.tgz",
+      "integrity": "sha512-09eO4QiXNFXJffJaOW5P6x6F5RLihpLUkXttvUZeWml0fU6x6Zp7AjG9zaeMpgH2ZNvq4GR1ytB22ddYcqJIZA==",
+      "dependencies": {
+        "find-exec": "1.0.3"
+      }
+    },
     "node_modules/possible-typed-array-names": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz",
@@ -13498,7 +13515,6 @@
       "version": "1.8.2",
       "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.8.2.tgz",
       "integrity": "sha512-AzqKpGKjrj7EM6rKVQEPpB288oCfnrEIuyoT9cyF4nmGa7V8Zk6f7RRqYisX8X9m+Q7bd632aZW4ky7EhbQztA==",
-      "dev": true,
       "license": "MIT",
       "engines": {
         "node": ">= 0.4"

package.json

@@ -195,6 +195,7 @@
     "os-name": "^6.0.0",
     "p-wait-for": "^5.0.2",
     "pdf-parse": "^1.1.1",
+    "play-sound": "^1.1.6",
     "puppeteer-chromium-resolver": "^23.0.0",
     "puppeteer-core": "^23.4.0",
     "serialize-error": "^11.0.3",

src/core/webview/ClineProvider.ts

@@ -20,6 +20,7 @@ import { Cline } from "../Cline"
 import { openMention } from "../mentions"
 import { getNonce } from "./getNonce"
 import { getUri } from "./getUri"
+import { playSound, setSoundEnabled } from "../../utils/sound"
 
 /*
 https://github.com/microsoft/vscode-webview-ui-toolkit-samples/blob/main/default/weather-webview/src/providers/WeatherViewProvider.ts
@@ -61,6 +62,7 @@ type GlobalStateKey =
 	| "openRouterModelId"
 	| "openRouterModelInfo"
 	| "allowedCommands"
+	| "soundEnabled"
 
 export const GlobalFileNames = {
 	apiConversationHistory: "api_conversation_history.json",
@@ -520,6 +522,18 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 						break;
 					// Add more switch case statements here as more webview message commands
 					// are created within the webview context (i.e. inside media/main.js)
+					case "playSound":
+						if (message.audioType) {
+							const soundPath = path.join(this.context.extensionPath, "audio", `${message.audioType}.wav`)
+							playSound(soundPath)
+						}
+						break
+					case "soundEnabled":
+						const enabled = message.bool ?? true
+						await this.updateGlobalState("soundEnabled", enabled)
+						setSoundEnabled(enabled)
+						await this.postStateToWebview()
+						break
 				}
 			},
 			null,
@@ -825,6 +839,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 			alwaysAllowWrite,
 			alwaysAllowExecute,
 			alwaysAllowBrowser,
+			soundEnabled,
 			taskHistory,
 		} = await this.getState()
 		
@@ -845,6 +860,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 			taskHistory: (taskHistory || [])
 				.filter((item) => item.ts && item.task)
 				.sort((a, b) => b.ts - a.ts),
+			soundEnabled: soundEnabled ?? true,
 			shouldShowAnnouncement: lastShownAnnouncementId !== this.latestAnnouncementId,
 			allowedCommands,
 		}
@@ -935,6 +951,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 			alwaysAllowBrowser,
 			taskHistory,
 			allowedCommands,
+			soundEnabled,
 		] = await Promise.all([
 			this.getGlobalState("apiProvider") as Promise<ApiProvider | undefined>,
 			this.getGlobalState("apiModelId") as Promise<string | undefined>,
@@ -968,6 +985,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 			this.getGlobalState("alwaysAllowBrowser") as Promise<boolean | undefined>,
 			this.getGlobalState("taskHistory") as Promise<HistoryItem[] | undefined>,
 			this.getGlobalState("allowedCommands") as Promise<string[] | undefined>,
+			this.getGlobalState("soundEnabled") as Promise<boolean | undefined>,
 		])
 
 		let apiProvider: ApiProvider
@@ -1019,6 +1037,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 			alwaysAllowBrowser: alwaysAllowBrowser ?? false,
 			taskHistory,
 			allowedCommands,
+			soundEnabled,
 		}
 	}
 

src/shared/ExtensionMessage.ts

@@ -41,6 +41,7 @@ export interface ExtensionState {
 	alwaysAllowBrowser?: boolean
 	uriScheme?: string
 	allowedCommands?: string[]
+	soundEnabled?: boolean
 }
 
 export interface ClineMessage {

src/shared/WebviewMessage.ts

@@ -1,5 +1,7 @@
 import { ApiConfiguration, ApiProvider } from "./api"
 
+export type AudioType = "notification" | "celebration" | "progress_loop"
+
 export interface WebviewMessage {
 	type:
 		| "apiConfiguration"
@@ -27,12 +29,15 @@ export interface WebviewMessage {
 		| "cancelTask"
 		| "refreshOpenRouterModels"
 		| "alwaysAllowBrowser"
+		| "playSound"
+		| "soundEnabled"
 	text?: string
 	askResponse?: ClineAskResponse
 	apiConfiguration?: ApiConfiguration
 	images?: string[]
 	bool?: boolean
 	commands?: string[]
+	audioType?: AudioType
 }
 
 export type ClineAskResponse = "yesButtonClicked" | "noButtonClicked" | "messageResponse"

src/utils/sound.ts

@@ -0,0 +1,68 @@
+import * as vscode from "vscode"
+import * as path from "path"
+
+/**
+ * Minimum interval (in milliseconds) to prevent continuous playback
+ */
+const MIN_PLAY_INTERVAL = 500
+
+/**
+ * Timestamp of when sound was last played
+ */
+let lastPlayedTime = 0
+
+/**
+ * Determine if a file is a WAV file
+ * @param filepath string
+ * @returns boolean
+ */
+export const isWAV = (filepath: string): boolean => {
+	return path.extname(filepath).toLowerCase() === ".wav"
+}
+
+let isSoundEnabled = true
+
+/**
+ * Set sound configuration
+ * @param enabled boolean
+ */
+export const setSoundEnabled = (enabled: boolean): void => {
+	isSoundEnabled = enabled
+}
+
+/**
+ * Play a sound file
+ * @param filepath string
+ * @return void
+ */
+export const playSound = (filepath: string): void => {
+	try {
+		if (!isSoundEnabled) {
+			return
+		}
+
+		if (!filepath) {
+			return
+		}
+
+		if (!isWAV(filepath)) {
+			throw new Error("Only wav files are supported.")
+		}
+
+		const currentTime = Date.now()
+		if (currentTime - lastPlayedTime < MIN_PLAY_INTERVAL) {
+			return // Skip playback within minimum interval to prevent continuous playback
+		}
+
+		const player = require("play-sound")()
+		player.play(filepath, function (err: any) {
+			if (err) {
+				throw new Error("Failed to play sound effect")
+			}
+		})
+
+		lastPlayedTime = currentTime
+	} catch (error: any) {
+		vscode.window.showErrorMessage(error.message)
+	}
+}

webview-ui/src/components/chat/ChatView.tsx

@@ -24,6 +24,7 @@ import BrowserSessionRow from "./BrowserSessionRow"
 import ChatRow from "./ChatRow"
 import ChatTextArea from "./ChatTextArea"
 import TaskHeader from "./TaskHeader"
+import { AudioType } from "../../../../src/shared/WebviewMessage"
 
 interface ChatViewProps {
 	isHidden: boolean
@@ -61,10 +62,24 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 	const [showScrollToBottom, setShowScrollToBottom] = useState(false)
 	const [isAtBottom, setIsAtBottom] = useState(false)
 
+	const [wasStreaming, setWasStreaming] = useState<boolean>(false)
+	const [hasStarted, setHasStarted] = useState(false)
+
 	// UI layout depends on the last 2 messages
 	// (since it relies on the content of these messages, we are deep comparing. i.e. the button state after hitting button sets enableButtons to false, and this effect otherwise would have to true again even if messages didn't change
 	const lastMessage = useMemo(() => messages.at(-1), [messages])
 	const secondLastMessage = useMemo(() => messages.at(-2), [messages])
+
+	function playSound(audioType: AudioType) {
+		vscode.postMessage({ type: "playSound", audioType })
+	}
+
+	function playSoundOnMessage(audioType: AudioType) {
+		if (hasStarted && !isStreaming) {
+			playSound(audioType)
+		}
+	}
+
 	useDeepCompareEffect(() => {
 		// if last message is an ask, show user ask UI
 		// if user finished a task, then start a new task with a new conversation history since in this moment that the extension is waiting for user response, the user could close the extension and the conversation history would be lost.
@@ -75,6 +90,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 					const isPartial = lastMessage.partial === true
 					switch (lastMessage.ask) {
 						case "api_req_failed":
+							playSoundOnMessage("progress_loop")
 							setTextAreaDisabled(true)
 							setClineAsk("api_req_failed")
 							setEnableButtons(true)
@@ -82,6 +98,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 							setSecondaryButtonText("Start New Task")
 							break
 						case "mistake_limit_reached":
+							playSoundOnMessage("progress_loop")
 							setTextAreaDisabled(false)
 							setClineAsk("mistake_limit_reached")
 							setEnableButtons(true)
@@ -89,6 +106,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 							setSecondaryButtonText("Start New Task")
 							break
 						case "followup":
+							playSoundOnMessage("notification")
 							setTextAreaDisabled(isPartial)
 							setClineAsk("followup")
 							setEnableButtons(isPartial)
@@ -96,6 +114,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 							// setSecondaryButtonText(undefined)
 							break
 						case "tool":
+							playSoundOnMessage("notification")
 							setTextAreaDisabled(isPartial)
 							setClineAsk("tool")
 							setEnableButtons(!isPartial)
@@ -113,6 +132,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 							}
 							break
 						case "browser_action_launch":
+							playSoundOnMessage("notification")
 							setTextAreaDisabled(isPartial)
 							setClineAsk("browser_action_launch")
 							setEnableButtons(!isPartial)
@@ -120,6 +140,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 							setSecondaryButtonText("Reject")
 							break
 						case "command":
+							playSoundOnMessage("notification")
 							setTextAreaDisabled(isPartial)
 							setClineAsk("command")
 							setEnableButtons(!isPartial)
@@ -127,6 +148,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 							setSecondaryButtonText("Reject")
 							break
 						case "command_output":
+							playSoundOnMessage("notification")
 							setTextAreaDisabled(false)
 							setClineAsk("command_output")
 							setEnableButtons(true)
@@ -135,6 +157,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 							break
 						case "completion_result":
 							// extension waiting for feedback. but we can just present a new task button
+							playSoundOnMessage("celebration")
 							setTextAreaDisabled(isPartial)
 							setClineAsk("completion_result")
 							setEnableButtons(!isPartial)
@@ -142,6 +165,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 							setSecondaryButtonText(undefined)
 							break
 						case "resume_task":
+							playSoundOnMessage("notification")
 							setTextAreaDisabled(false)
 							setClineAsk("resume_task")
 							setEnableButtons(true)
@@ -150,6 +174,7 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 							setDidClickCancel(false) // special case where we reset the cancel button state							
 							break
 						case "resume_completed_task":
+							playSoundOnMessage("celebration")
 							setTextAreaDisabled(false)
 							setClineAsk("resume_completed_task")
 							setEnableButtons(true)
@@ -441,6 +466,36 @@ const ChatView = ({ isHidden, showAnnouncement, hideAnnouncement, showHistoryVie
 			return true
 		})
 	}, [modifiedMessages])
+	useEffect(() => {
+		if (isStreaming) {
+			// Set to true once any request has started
+			setHasStarted(true)
+		}
+		// Only execute when isStreaming changes from true to false
+		if (wasStreaming && !isStreaming && lastMessage) {
+			// Play appropriate sound based on lastMessage content
+			if (lastMessage.type === "ask") {
+				switch (lastMessage.ask) {
+					case "api_req_failed":
+					case "mistake_limit_reached":
+						playSound("progress_loop")
+						break
+					case "tool":
+					case "followup":
+					case "browser_action_launch":
+					case "resume_task":
+						playSound("notification")
+						break
+					case "completion_result":
+					case "resume_completed_task":
+						playSound("celebration")
+						break
+				}
+			}
+		}
+		// Update previous value
+		setWasStreaming(isStreaming)
+	}, [isStreaming, lastMessage])
 
 	const isBrowserSessionMessage = (message: ClineMessage): boolean => {
 		// which of visible messages are browser session messages, see above

webview-ui/src/components/settings/SettingsView.tsx

@@ -25,6 +25,8 @@ const SettingsView = ({ onDone }: SettingsViewProps) => {
 		setAlwaysAllowExecute,
 		alwaysAllowBrowser,
 		setAlwaysAllowBrowser,
+		soundEnabled,
+		setSoundEnabled,
 		openRouterModels,
 		setAllowedCommands,
 		allowedCommands,
@@ -47,6 +49,7 @@ const SettingsView = ({ onDone }: SettingsViewProps) => {
 			vscode.postMessage({ type: "alwaysAllowExecute", bool: alwaysAllowExecute })
 			vscode.postMessage({ type: "alwaysAllowBrowser", bool: alwaysAllowBrowser })
 			vscode.postMessage({ type: "allowedCommands", commands: allowedCommands ?? [] })
+			vscode.postMessage({ type: "soundEnabled", bool: soundEnabled })
 			onDone()
 		}
 	}
@@ -285,6 +288,20 @@ const SettingsView = ({ onDone }: SettingsViewProps) => {
 					</div>
 				)}
 
+				<div style={{ marginBottom: 5 }}>
+					<VSCodeCheckbox checked={soundEnabled} onChange={(e: any) => setSoundEnabled(e.target.checked)}>
+						<span style={{ fontWeight: "500" }}>Enable sound effects</span>
+					</VSCodeCheckbox>
+					<p
+						style={{
+							fontSize: "12px",
+							marginTop: "5px",
+							color: "var(--vscode-descriptionForeground)",
+						}}>
+						When enabled, Cline will play sound effects for notifications and events.
+					</p>
+				</div>
+
 				{IS_DEV && (
 					<>
 						<div style={{ marginTop: "10px", marginBottom: "4px" }}>Debug</div>

webview-ui/src/components/settings/__tests__/SettingsView.test.tsx

@@ -92,6 +92,44 @@ const renderSettingsView = () => {
   return { onDone }
 }
 
+describe('SettingsView - Sound Settings', () => {
+  beforeEach(() => {
+    jest.clearAllMocks()
+  })
+
+  it('initializes with sound disabled by default', () => {
+    renderSettingsView()
+    
+    const soundCheckbox = screen.getByRole('checkbox', {
+      name: /Enable sound effects/i
+    })
+    expect(soundCheckbox).not.toBeChecked()
+  })
+
+  it('toggles sound setting and sends message to VSCode', () => {
+    renderSettingsView()
+    
+    const soundCheckbox = screen.getByRole('checkbox', {
+      name: /Enable sound effects/i
+    })
+    
+    // Enable sound
+    fireEvent.click(soundCheckbox)
+    expect(soundCheckbox).toBeChecked()
+    
+    // Click Done to save settings
+    const doneButton = screen.getByText('Done')
+    fireEvent.click(doneButton)
+    
+    expect(vscode.postMessage).toHaveBeenCalledWith(
+      expect.objectContaining({
+        type: 'soundEnabled',
+        bool: true
+      })
+    )
+  })
+})
+
 describe('SettingsView - Allowed Commands', () => {
   beforeEach(() => {
     jest.clearAllMocks()

webview-ui/src/context/ExtensionStateContext.tsx

@@ -25,6 +25,7 @@ export interface ExtensionStateContextType extends ExtensionState {
 	setAlwaysAllowBrowser: (value: boolean) => void
 	setShowAnnouncement: (value: boolean) => void
 	setAllowedCommands: (value: string[]) => void
+	setSoundEnabled: (value: boolean) => void
 }
 
 const ExtensionStateContext = createContext<ExtensionStateContextType | undefined>(undefined)
@@ -36,6 +37,7 @@ export const ExtensionStateContextProvider: React.FC<{ children: React.ReactNode
 		taskHistory: [],
 		shouldShowAnnouncement: false,
 		allowedCommands: [],
+		soundEnabled: false,
 	})
 	const [didHydrateState, setDidHydrateState] = useState(false)
 	const [showWelcome, setShowWelcome] = useState(false)
@@ -124,6 +126,7 @@ export const ExtensionStateContextProvider: React.FC<{ children: React.ReactNode
 		setAlwaysAllowBrowser: (value) => setState((prevState) => ({ ...prevState, alwaysAllowBrowser: value })),
 		setShowAnnouncement: (value) => setState((prevState) => ({ ...prevState, shouldShowAnnouncement: value })),
 		setAllowedCommands: (value) => setState((prevState) => ({ ...prevState, allowedCommands: value })),
+		setSoundEnabled: (value) => setState((prevState) => ({ ...prevState, soundEnabled: value })),
 	}
 
 	return <ExtensionStateContext.Provider value={contextValue}>{children}</ExtensionStateContext.Provider>

webview-ui/src/context/__tests__/ExtensionStateContext.test.tsx

@@ -4,10 +4,11 @@ import { ExtensionStateContextProvider, useExtensionState } from '../ExtensionSt
 
 // Test component that consumes the context
 const TestComponent = () => {
-  const { allowedCommands, setAllowedCommands } = useExtensionState()
+  const { allowedCommands, setAllowedCommands, soundEnabled } = useExtensionState()
   return (
     <div>
       <div data-testid="allowed-commands">{JSON.stringify(allowedCommands)}</div>
+      <div data-testid="sound-enabled">{JSON.stringify(soundEnabled)}</div>
       <button
         data-testid="update-button"
         onClick={() => setAllowedCommands(['npm install', 'git status'])}
@@ -29,6 +30,16 @@ describe('ExtensionStateContext', () => {
     expect(JSON.parse(screen.getByTestId('allowed-commands').textContent!)).toEqual([])
   })
 
+  it('initializes with soundEnabled set to false', () => {
+    render(
+      <ExtensionStateContextProvider>
+        <TestComponent />
+      </ExtensionStateContextProvider>
+    )
+
+    expect(JSON.parse(screen.getByTestId('sound-enabled').textContent!)).toBe(false)
+  })
+
   it('updates allowedCommands through setAllowedCommands', () => {
     render(
       <ExtensionStateContextProvider>