From 8f0548ec97a12899a9d67c2c8242616569c19d6c Mon Sep 17 00:00:00 2001 From: pacnpal <183241239+pacnpal@users.noreply.github.com> Date: Mon, 3 Feb 2025 13:08:13 -0500 Subject: [PATCH] feat: add Azure AI models support in context and API handling --- src/core/webview/ClineProvider.ts | 31 ++ src/shared/ExtensionMessage.ts | 363 +++++++------- src/shared/checkExistApiConfig.ts | 1 + .../settings/AzureAiModelPicker.tsx | 469 +++++++++++++----- .../src/context/ExtensionStateContext.tsx | 3 + 5 files changed, 574 insertions(+), 293 deletions(-) diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index d0228dc..30dde13 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -754,6 +754,15 @@ export class ClineProvider implements vscode.WebviewViewProvider { this.postMessageToWebview({ type: "openAiModels", openAiModels }) } break + case "refreshAzureAiModels": + if (message?.values?.endpoint && message?.values?.key) { + const azureAiModels = await this.getAzureAiModels( + message?.values?.endpoint, + message?.values?.key, + ) + this.postMessageToWebview({ type: "azureAiModels", azureAiModels }) + } + break case "openImage": openImage(message.text!) break @@ -1645,6 +1654,28 @@ export class ClineProvider implements vscode.WebviewViewProvider { } } + // Azure AI + + async getAzureAiModels(endpoint?: string, key?: string) { + try { + if (!endpoint || !key) { + return [] + } + + if (!URL.canParse(endpoint)) { + return [] + } + + const response = await axios.get(`${endpoint}/deployments`, { + headers: { "api-key": key }, + }) + const modelsArray = response.data?.value?.map((deployment: any) => deployment.name) || [] + const models = [...new Set(modelsArray)] + return models + } catch (error) { + return [] + } + } // OpenRouter async handleOpenRouterCallback(code: string) { diff --git a/src/shared/ExtensionMessage.ts b/src/shared/ExtensionMessage.ts index 9bb0f48..26fc8bc 100644 --- a/src/shared/ExtensionMessage.ts +++ b/src/shared/ExtensionMessage.ts @@ -7,220 +7,223 @@ import { CustomSupportPrompts } from "./support-prompt" import { ExperimentId } from "./experiments" export interface LanguageModelChatSelector { - vendor?: string - family?: string - version?: string - id?: string + vendor?: string + family?: string + version?: string + id?: string } export interface ExtensionMessage { - type: - | "action" - | "state" - | "selectedImages" - | "ollamaModels" - | "lmStudioModels" - | "theme" - | "workspaceUpdated" - | "invoke" - | "partialMessage" - | "glamaModels" - | "openRouterModels" - | "openAiModels" - | "mcpServers" - | "enhancedPrompt" - | "commitSearchResults" - | "listApiConfig" - | "vsCodeLmModels" - | "vsCodeLmApiAvailable" - | "requestVsCodeLmModels" - | "updatePrompt" - | "systemPrompt" - | "autoApprovalEnabled" - | "updateCustomMode" - | "deleteCustomMode" - text?: string - action?: - | "chatButtonClicked" - | "mcpButtonClicked" - | "settingsButtonClicked" - | "historyButtonClicked" - | "promptsButtonClicked" - | "didBecomeVisible" - invoke?: "sendMessage" | "primaryButtonClick" | "secondaryButtonClick" | "setChatBoxMessage" - state?: ExtensionState - images?: string[] - ollamaModels?: string[] - lmStudioModels?: string[] - vsCodeLmModels?: { vendor?: string; family?: string; version?: string; id?: string }[] - filePaths?: string[] - openedTabs?: Array<{ - label: string - isActive: boolean - path?: string - }> - partialMessage?: ClineMessage - glamaModels?: Record - openRouterModels?: Record - openAiModels?: string[] - mcpServers?: McpServer[] - commits?: GitCommit[] - listApiConfig?: ApiConfigMeta[] - mode?: Mode - customMode?: ModeConfig - slug?: string + type: + | "action" + | "state" + | "selectedImages" + | "ollamaModels" + | "lmStudioModels" + | "theme" + | "workspaceUpdated" + | "invoke" + | "partialMessage" + | "glamaModels" + | "openRouterModels" + | "openAiModels" + | "refreshAzureAiModels" + | "azureAiModels" + | "mcpServers" + | "enhancedPrompt" + | "commitSearchResults" + | "listApiConfig" + | "vsCodeLmModels" + | "vsCodeLmApiAvailable" + | "requestVsCodeLmModels" + | "updatePrompt" + | "systemPrompt" + | "autoApprovalEnabled" + | "updateCustomMode" + | "deleteCustomMode" + text?: string + action?: + | "chatButtonClicked" + | "mcpButtonClicked" + | "settingsButtonClicked" + | "historyButtonClicked" + | "promptsButtonClicked" + | "didBecomeVisible" + invoke?: "sendMessage" | "primaryButtonClick" | "secondaryButtonClick" | "setChatBoxMessage" + state?: ExtensionState + images?: string[] + ollamaModels?: string[] + lmStudioModels?: string[] + vsCodeLmModels?: { vendor?: string; family?: string; version?: string; id?: string }[] + filePaths?: string[] + openedTabs?: Array<{ + label: string + isActive: boolean + path?: string + }> + partialMessage?: ClineMessage + glamaModels?: Record + openRouterModels?: Record + openAiModels?: string[] + azureAiModels?: string[] + mcpServers?: McpServer[] + commits?: GitCommit[] + listApiConfig?: ApiConfigMeta[] + mode?: Mode + customMode?: ModeConfig + slug?: string } export interface ApiConfigMeta { - id: string - name: string - apiProvider?: ApiProvider + id: string + name: string + apiProvider?: ApiProvider } export interface ExtensionState { - version: string - clineMessages: ClineMessage[] - taskHistory: HistoryItem[] - shouldShowAnnouncement: boolean - apiConfiguration?: ApiConfiguration - currentApiConfigName?: string - listApiConfigMeta?: ApiConfigMeta[] - customInstructions?: string - customModePrompts?: CustomModePrompts - customSupportPrompts?: CustomSupportPrompts - alwaysAllowReadOnly?: boolean - alwaysAllowWrite?: boolean - alwaysAllowExecute?: boolean - alwaysAllowBrowser?: boolean - alwaysAllowMcp?: boolean - alwaysApproveResubmit?: boolean - alwaysAllowModeSwitch?: boolean - requestDelaySeconds: number - rateLimitSeconds: number - uriScheme?: string - allowedCommands?: string[] - soundEnabled?: boolean - soundVolume?: number - diffEnabled?: boolean - browserViewportSize?: string - screenshotQuality?: number - fuzzyMatchThreshold?: number - preferredLanguage: string - writeDelayMs: number - terminalOutputLineLimit?: number - mcpEnabled: boolean - enableMcpServerCreation: boolean - mode: Mode - modeApiConfigs?: Record - enhancementApiConfigId?: string - experiments: Record - autoApprovalEnabled?: boolean - customModes: ModeConfig[] - toolRequirements?: Record - azureAiDeployments?: Record + version: string + clineMessages: ClineMessage[] + taskHistory: HistoryItem[] + shouldShowAnnouncement: boolean + apiConfiguration?: ApiConfiguration + currentApiConfigName?: string + listApiConfigMeta?: ApiConfigMeta[] + customInstructions?: string + customModePrompts?: CustomModePrompts + customSupportPrompts?: CustomSupportPrompts + alwaysAllowReadOnly?: boolean + alwaysAllowWrite?: boolean + alwaysAllowExecute?: boolean + alwaysAllowBrowser?: boolean + alwaysAllowMcp?: boolean + alwaysApproveResubmit?: boolean + alwaysAllowModeSwitch?: boolean + requestDelaySeconds: number + rateLimitSeconds: number + uriScheme?: string + allowedCommands?: string[] + soundEnabled?: boolean + soundVolume?: number + diffEnabled?: boolean + browserViewportSize?: string + screenshotQuality?: number + fuzzyMatchThreshold?: number + preferredLanguage: string + writeDelayMs: number + terminalOutputLineLimit?: number + mcpEnabled: boolean + enableMcpServerCreation: boolean + mode: Mode + modeApiConfigs?: Record + enhancementApiConfigId?: string + experiments: Record + autoApprovalEnabled?: boolean + customModes: ModeConfig[] + toolRequirements?: Record + azureAiDeployments?: Record } export interface ClineMessage { - ts: number - type: "ask" | "say" - ask?: ClineAsk - say?: ClineSay - text?: string - images?: string[] - partial?: boolean - reasoning?: string + ts: number + type: "ask" | "say" + ask?: ClineAsk + say?: ClineSay + text?: string + images?: string[] + partial?: boolean + reasoning?: string } export type ClineAsk = - | "followup" - | "command" - | "command_output" - | "completion_result" - | "tool" - | "api_req_failed" - | "resume_task" - | "resume_completed_task" - | "mistake_limit_reached" - | "browser_action_launch" - | "use_mcp_server" + | "followup" + | "command" + | "command_output" + | "completion_result" + | "tool" + | "api_req_failed" + | "resume_task" + | "resume_completed_task" + | "mistake_limit_reached" + | "browser_action_launch" + | "use_mcp_server" export type ClineSay = - | "task" - | "error" - | "api_req_started" - | "api_req_finished" - | "text" - | "reasoning" - | "completion_result" - | "user_feedback" - | "user_feedback_diff" - | "api_req_retried" - | "api_req_retry_delayed" - | "command_output" - | "tool" - | "shell_integration_warning" - | "browser_action" - | "browser_action_result" - | "command" - | "mcp_server_request_started" - | "mcp_server_response" - | "new_task_started" - | "new_task" + | "task" + | "error" + | "api_req_started" + | "api_req_finished" + | "text" + | "reasoning" + | "completion_result" + | "user_feedback" + | "user_feedback_diff" + | "api_req_retried" + | "api_req_retry_delayed" + | "command_output" + | "tool" + | "shell_integration_warning" + | "browser_action" + | "browser_action_result" + | "command" + | "mcp_server_request_started" + | "mcp_server_response" + | "new_task_started" + | "new_task" export interface ClineSayTool { - tool: - | "editedExistingFile" - | "appliedDiff" - | "newFileCreated" - | "readFile" - | "listFilesTopLevel" - | "listFilesRecursive" - | "listCodeDefinitionNames" - | "searchFiles" - | "switchMode" - | "newTask" - path?: string - diff?: string - content?: string - regex?: string - filePattern?: string - mode?: string - reason?: string + tool: + | "editedExistingFile" + | "appliedDiff" + | "newFileCreated" + | "readFile" + | "listFilesTopLevel" + | "listFilesRecursive" + | "listCodeDefinitionNames" + | "searchFiles" + | "switchMode" + | "newTask" + path?: string + diff?: string + content?: string + regex?: string + filePattern?: string + mode?: string + reason?: string } export const browserActions = ["launch", "click", "type", "scroll_down", "scroll_up", "close"] as const export type BrowserAction = (typeof browserActions)[number] export interface ClineSayBrowserAction { - action: BrowserAction - coordinate?: string - text?: string + action: BrowserAction + coordinate?: string + text?: string } export type BrowserActionResult = { - screenshot?: string - logs?: string - currentUrl?: string - currentMousePosition?: string + screenshot?: string + logs?: string + currentUrl?: string + currentMousePosition?: string } export interface ClineAskUseMcpServer { - serverName: string - type: "use_mcp_tool" | "access_mcp_resource" - toolName?: string - arguments?: string - uri?: string + serverName: string + type: "use_mcp_tool" | "access_mcp_resource" + toolName?: string + arguments?: string + uri?: string } export interface ClineApiReqInfo { - request?: string - tokensIn?: number - tokensOut?: number - cacheWrites?: number - cacheReads?: number - cost?: number - cancelReason?: ClineApiReqCancelReason - streamingFailedMessage?: string + request?: string + tokensIn?: number + tokensOut?: number + cacheWrites?: number + cacheReads?: number + cost?: number + cancelReason?: ClineApiReqCancelReason + streamingFailedMessage?: string } export type ClineApiReqCancelReason = "streaming_failed" | "user_cancelled" diff --git a/src/shared/checkExistApiConfig.ts b/src/shared/checkExistApiConfig.ts index 8cb8055..03d1c06 100644 --- a/src/shared/checkExistApiConfig.ts +++ b/src/shared/checkExistApiConfig.ts @@ -16,6 +16,7 @@ export function checkExistKey(config: ApiConfiguration | undefined) { config.deepSeekApiKey, config.mistralApiKey, config.vsCodeLmModelSelector, + config.azureAiKey, ].some((key) => key !== undefined) : false } diff --git a/webview-ui/src/components/settings/AzureAiModelPicker.tsx b/webview-ui/src/components/settings/AzureAiModelPicker.tsx index ea7d058..e521003 100644 --- a/webview-ui/src/components/settings/AzureAiModelPicker.tsx +++ b/webview-ui/src/components/settings/AzureAiModelPicker.tsx @@ -1,100 +1,305 @@ import { VSCodeLink, VSCodeTextField } from "@vscode/webview-ui-toolkit/react" -import { memo } from "react" +import { Fzf } from "fzf" +import React, { KeyboardEvent, memo, useEffect, useMemo, useRef, useState } from "react" +import debounce from "debounce" import { useExtensionState } from "../../context/ExtensionStateContext" +import { vscode } from "../../utils/vscode" +import { highlightFzfMatch } from "../../utils/highlight" import { Pane } from "vscrui" import { azureAiModelInfoSaneDefaults } from "../../../../src/shared/api" +import styled from "styled-components" const AzureAiModelPicker: React.FC = () => { - const { apiConfiguration, handleInputChange } = useExtensionState() + const { apiConfiguration, setApiConfiguration, azureAiModels, onUpdateApiConfig } = useExtensionState() + const [searchTerm, setSearchTerm] = useState(apiConfiguration?.apiModelId || "") + const [isDropdownVisible, setIsDropdownVisible] = useState(false) + const [selectedIndex, setSelectedIndex] = useState(-1) + const dropdownRef = useRef(null) + const itemRefs = useRef<(HTMLDivElement | null)[]>([]) + const dropdownListRef = useRef(null) + + const handleModelChange = (newModelId: string) => { + const apiConfig = { + ...apiConfiguration, + apiModelId: newModelId, + } + setApiConfiguration(apiConfig) + onUpdateApiConfig(apiConfig) + setSearchTerm(newModelId) + } + + useEffect(() => { + if (apiConfiguration?.apiModelId && apiConfiguration?.apiModelId !== searchTerm) { + setSearchTerm(apiConfiguration?.apiModelId) + } + }, [apiConfiguration, searchTerm]) + + const debouncedRefreshModels = useMemo( + () => + debounce((endpoint: string, key: string) => { + vscode.postMessage({ + type: "refreshAzureAiModels", + values: { + endpoint, + key, + }, + }) + }, 50), + [], + ) + + useEffect(() => { + if (!apiConfiguration?.azureAiEndpoint || !apiConfiguration?.azureAiKey) { + return + } + + debouncedRefreshModels(apiConfiguration.azureAiEndpoint, apiConfiguration.azureAiKey) + + return () => { + debouncedRefreshModels.clear() + } + }, [apiConfiguration?.azureAiEndpoint, apiConfiguration?.azureAiKey, debouncedRefreshModels]) + + useEffect(() => { + const handleClickOutside = (event: MouseEvent) => { + if (dropdownRef.current && !dropdownRef.current.contains(event.target as Node)) { + setIsDropdownVisible(false) + } + } + + document.addEventListener("mousedown", handleClickOutside) + return () => { + document.removeEventListener("mousedown", handleClickOutside) + } + }, []) + + const modelIds = useMemo(() => { + return azureAiModels.sort((a, b) => a.localeCompare(b)) + }, [azureAiModels]) + + const searchableItems = useMemo(() => { + return modelIds.map((id) => ({ + id, + html: id, + })) + }, [modelIds]) + + const fzf = useMemo(() => { + return new Fzf(searchableItems, { + selector: (item) => item.html, + }) + }, [searchableItems]) + + const modelSearchResults = useMemo(() => { + if (!searchTerm) return searchableItems + + const searchResults = fzf.find(searchTerm) + return searchResults.map((result) => ({ + ...result.item, + html: highlightFzfMatch(result.item.html, Array.from(result.positions), "model-item-highlight"), + })) + }, [searchableItems, searchTerm, fzf]) + + const handleKeyDown = (event: KeyboardEvent) => { + if (!isDropdownVisible) return + + switch (event.key) { + case "ArrowDown": + event.preventDefault() + setSelectedIndex((prev) => (prev < modelSearchResults.length - 1 ? prev + 1 : prev)) + break + case "ArrowUp": + event.preventDefault() + setSelectedIndex((prev) => (prev > 0 ? prev - 1 : prev)) + break + case "Enter": + event.preventDefault() + if (selectedIndex >= 0 && selectedIndex < modelSearchResults.length) { + handleModelChange(modelSearchResults[selectedIndex].id) + setIsDropdownVisible(false) + } + break + case "Escape": + setIsDropdownVisible(false) + setSelectedIndex(-1) + break + } + } + + useEffect(() => { + setSelectedIndex(-1) + if (dropdownListRef.current) { + dropdownListRef.current.scrollTop = 0 + } + }, [searchTerm]) + + useEffect(() => { + if (selectedIndex >= 0 && itemRefs.current[selectedIndex]) { + itemRefs.current[selectedIndex]?.scrollIntoView({ + block: "nearest", + behavior: "smooth", + }) + } + }, [selectedIndex]) return ( -
- - Base URL - + <> + +
+ { + const apiConfig = { + ...apiConfiguration, + azureAiEndpoint: (e.target as HTMLInputElement).value, + } + setApiConfiguration(apiConfig) + onUpdateApiConfig(apiConfig) + }} + placeholder="https://your-endpoint.region.inference.ai.azure.com"> + Base URL + - - API Key - + { + const apiConfig = { + ...apiConfiguration, + azureAiKey: (e.target as HTMLInputElement).value, + } + setApiConfiguration(apiConfig) + onUpdateApiConfig(apiConfig) + }} + placeholder="Enter API Key..."> + API Key + - - Model Deployment Name - - - - handleInputChange("azureAiModelConfig")({ - target: { value: azureAiModelInfoSaneDefaults }, - }), - }, - ]}> -
-

- Configure capabilities for your deployed model. -

+ + { + handleModelChange((e.target as HTMLInputElement)?.value) + setIsDropdownVisible(true) + }} + onFocus={() => setIsDropdownVisible(true)} + onKeyDown={handleKeyDown} + style={{ width: "100%", zIndex: AZURE_MODEL_PICKER_Z_INDEX, position: "relative" }}> + Model Deployment Name + {searchTerm && ( +
{ + handleModelChange("") + setIsDropdownVisible(true) + }} + slot="end" + style={{ + display: "flex", + justifyContent: "center", + alignItems: "center", + height: "100%", + }} + /> + )} + + {isDropdownVisible && ( + + {modelSearchResults.map((item, index) => ( + (itemRefs.current[index] = el)} + isSelected={index === selectedIndex} + onMouseEnter={() => setSelectedIndex(index)} + onClick={() => { + handleModelChange(item.id) + setIsDropdownVisible(false) + }} + dangerouslySetInnerHTML={{ + __html: item.html, + }} + /> + ))} + + )} + + { + const apiConfig = { + ...apiConfiguration, + azureAiModelConfig: azureAiModelInfoSaneDefaults, + } + setApiConfiguration(apiConfig) + onUpdateApiConfig(apiConfig) + }, + }, + ]}>
- - Model Features - + Configure capabilities for your deployed model. +

-
- { - const parsed = parseInt(e.target.value) - handleInputChange("azureAiModelConfig")({ - target: { - value: { +
+ + Model Features + + +
+ { + const parsed = parseInt(e.target.value) + const apiConfig = { + ...apiConfiguration, + azureAiModelConfig: { ...(apiConfiguration?.azureAiModelConfig || azureAiModelInfoSaneDefaults), contextWindow: @@ -104,43 +309,81 @@ const AzureAiModelPicker: React.FC = () => { ? azureAiModelInfoSaneDefaults.contextWindow : parsed, }, - }, - }) - }} - placeholder="e.g. 128000"> - Context Window Size - -

- Total tokens the model can process in a single request. -

+ } + setApiConfiguration(apiConfig) + onUpdateApiConfig(apiConfig) + }} + placeholder="e.g. 128000"> + Context Window Size + +

+ Total tokens the model can process in a single request. +

+
-
- + -

- Configure your Azure AI Model Inference endpoint and model deployment. API keys are stored locally. - {!apiConfiguration?.azureAiKey && ( - - {" "} - Learn more about Azure AI Model Inference. - - )} -

-
+

+ Configure your Azure AI Model Inference endpoint and model deployment. API keys are stored locally. + {!apiConfiguration?.azureAiKey && ( + + {" "} + Learn more about Azure AI Model Inference. + + )} +

+
+ ) } export default memo(AzureAiModelPicker) + +// Dropdown + +const DropdownWrapper = styled.div` + position: relative; + width: 100%; +` + +export const AZURE_MODEL_PICKER_Z_INDEX = 1_000 + +const DropdownList = styled.div` + position: absolute; + top: calc(100% - 3px); + left: 0; + width: calc(100% - 2px); + max-height: 200px; + overflow-y: auto; + background-color: var(--vscode-dropdown-background); + border: 1px solid var(--vscode-list-activeSelectionBackground); + z-index: ${AZURE_MODEL_PICKER_Z_INDEX - 1}; + border-bottom-left-radius: 3px; + border-bottom-right-radius: 3px; +` + +const DropdownItem = styled.div<{ isSelected: boolean }>` + padding: 5px 10px; + cursor: pointer; + word-break: break-all; + white-space: normal; + + background-color: ${({ isSelected }) => (isSelected ? "var(--vscode-list-activeSelectionBackground)" : "inherit")}; + + &:hover { + background-color: var(--vscode-list-activeSelectionBackground); + } +` diff --git a/webview-ui/src/context/ExtensionStateContext.tsx b/webview-ui/src/context/ExtensionStateContext.tsx index 47db6bf..d6d548e 100644 --- a/webview-ui/src/context/ExtensionStateContext.tsx +++ b/webview-ui/src/context/ExtensionStateContext.tsx @@ -25,6 +25,7 @@ export interface ExtensionStateContextType extends ExtensionState { glamaModels: Record openRouterModels: Record openAiModels: string[] + azureAiModels: string[] mcpServers: McpServer[] filePaths: string[] openedTabs: Array<{ label: string; isActive: boolean; path?: string }> @@ -123,6 +124,7 @@ export const ExtensionStateContextProvider: React.FC<{ children: React.ReactNode }) const [openAiModels, setOpenAiModels] = useState([]) + const [azureAiModels, setAzureAiModels] = useState([]) const [mcpServers, setMcpServers] = useState([]) const setListApiConfigMeta = useCallback( @@ -247,6 +249,7 @@ export const ExtensionStateContextProvider: React.FC<{ children: React.ReactNode glamaModels, openRouterModels, openAiModels, + azureAiModels, mcpServers, filePaths, openedTabs,