Merge pull request #282 from RooVetGit/open_ai_streaming_toggle

Streaming checkbox for OpenAI-compatible providers

src/api/providers/__tests__/openai.test.ts

@@ -0,0 +1,192 @@
+import { OpenAiHandler } from '../openai'
+import { ApiHandlerOptions, openAiModelInfoSaneDefaults } from '../../../shared/api'
+import OpenAI, { AzureOpenAI } from 'openai'
+import { Anthropic } from '@anthropic-ai/sdk'
+
+// Mock dependencies
+jest.mock('openai')
+
+describe('OpenAiHandler', () => {
+    const mockOptions: ApiHandlerOptions = {
+        openAiApiKey: 'test-key',
+        openAiModelId: 'gpt-4',
+        openAiStreamingEnabled: true,
+        openAiBaseUrl: 'https://api.openai.com/v1'
+    }
+
+    beforeEach(() => {
+        jest.clearAllMocks()
+    })
+
+    test('constructor initializes with correct options', () => {
+        const handler = new OpenAiHandler(mockOptions)
+        expect(handler).toBeInstanceOf(OpenAiHandler)
+        expect(OpenAI).toHaveBeenCalledWith({
+            apiKey: mockOptions.openAiApiKey,
+            baseURL: mockOptions.openAiBaseUrl
+        })
+    })
+
+    test('constructor initializes Azure client when Azure URL is provided', () => {
+        const azureOptions: ApiHandlerOptions = {
+            ...mockOptions,
+            openAiBaseUrl: 'https://example.azure.com',
+            azureApiVersion: '2023-05-15'
+        }
+        const handler = new OpenAiHandler(azureOptions)
+        expect(handler).toBeInstanceOf(OpenAiHandler)
+        expect(AzureOpenAI).toHaveBeenCalledWith({
+            baseURL: azureOptions.openAiBaseUrl,
+            apiKey: azureOptions.openAiApiKey,
+            apiVersion: azureOptions.azureApiVersion
+        })
+    })
+
+    test('getModel returns correct model info', () => {
+        const handler = new OpenAiHandler(mockOptions)
+        const result = handler.getModel()
+        
+        expect(result).toEqual({
+            id: mockOptions.openAiModelId,
+            info: openAiModelInfoSaneDefaults
+        })
+    })
+
+    test('createMessage handles streaming correctly when enabled', async () => {
+        const handler = new OpenAiHandler({
+            ...mockOptions,
+            openAiStreamingEnabled: true,
+            includeMaxTokens: true
+        })
+        
+        const mockStream = {
+            async *[Symbol.asyncIterator]() {
+                yield {
+                    choices: [{
+                        delta: {
+                            content: 'test response'
+                        }
+                    }],
+                    usage: {
+                        prompt_tokens: 10,
+                        completion_tokens: 5
+                    }
+                }
+            }
+        }
+
+        const mockCreate = jest.fn().mockResolvedValue(mockStream)
+        ;(OpenAI as jest.MockedClass<typeof OpenAI>).prototype.chat = {
+            completions: { create: mockCreate }
+        } as any
+
+        const systemPrompt = 'test system prompt'
+        const messages: Anthropic.Messages.MessageParam[] = [
+            { role: 'user', content: 'test message' }
+        ]
+
+        const generator = handler.createMessage(systemPrompt, messages)
+        const chunks = []
+        
+        for await (const chunk of generator) {
+            chunks.push(chunk)
+        }
+
+        expect(chunks).toEqual([
+            {
+                type: 'text',
+                text: 'test response'
+            },
+            {
+                type: 'usage',
+                inputTokens: 10,
+                outputTokens: 5
+            }
+        ])
+
+        expect(mockCreate).toHaveBeenCalledWith({
+            model: mockOptions.openAiModelId,
+            messages: [
+                { role: 'system', content: systemPrompt },
+                { role: 'user', content: 'test message' }
+            ],
+            temperature: 0,
+            stream: true,
+            stream_options: { include_usage: true },
+            max_tokens: openAiModelInfoSaneDefaults.maxTokens
+        })
+    })
+
+    test('createMessage handles non-streaming correctly when disabled', async () => {
+        const handler = new OpenAiHandler({
+            ...mockOptions,
+            openAiStreamingEnabled: false
+        })
+        
+        const mockResponse = {
+            choices: [{
+                message: {
+                    content: 'test response'
+                }
+            }],
+            usage: {
+                prompt_tokens: 10,
+                completion_tokens: 5
+            }
+        }
+
+        const mockCreate = jest.fn().mockResolvedValue(mockResponse)
+        ;(OpenAI as jest.MockedClass<typeof OpenAI>).prototype.chat = {
+            completions: { create: mockCreate }
+        } as any
+
+        const systemPrompt = 'test system prompt'
+        const messages: Anthropic.Messages.MessageParam[] = [
+            { role: 'user', content: 'test message' }
+        ]
+
+        const generator = handler.createMessage(systemPrompt, messages)
+        const chunks = []
+        
+        for await (const chunk of generator) {
+            chunks.push(chunk)
+        }
+
+        expect(chunks).toEqual([
+            {
+                type: 'text',
+                text: 'test response'
+            },
+            {
+                type: 'usage',
+                inputTokens: 10,
+                outputTokens: 5
+            }
+        ])
+
+        expect(mockCreate).toHaveBeenCalledWith({
+            model: mockOptions.openAiModelId,
+            messages: [
+                { role: 'user', content: systemPrompt },
+                { role: 'user', content: 'test message' }
+            ]
+        })
+    })
+
+    test('createMessage handles API errors', async () => {
+        const handler = new OpenAiHandler(mockOptions)
+        const mockStream = {
+            async *[Symbol.asyncIterator]() {
+                throw new Error('API Error')
+            }
+        }
+
+        const mockCreate = jest.fn().mockResolvedValue(mockStream)
+        ;(OpenAI as jest.MockedClass<typeof OpenAI>).prototype.chat = {
+            completions: { create: mockCreate }
+        } as any
+
+        const generator = handler.createMessage('test', [])
+        await expect(generator.next()).rejects.toThrow('API Error')
+    })
+})

src/api/providers/openai.ts

@@ -32,42 +32,64 @@ export class OpenAiHandler implements ApiHandler {
 		}
 	}
 
-	// Include stream_options for OpenAI Compatible providers if the checkbox is checked
 	async *createMessage(systemPrompt: string, messages: Anthropic.Messages.MessageParam[]): ApiStream {
-		const openAiMessages: OpenAI.Chat.ChatCompletionMessageParam[] = [
-			{ role: "system", content: systemPrompt },
-			...convertToOpenAiMessages(messages),
-		]
 		const modelInfo = this.getModel().info
-		const requestOptions: OpenAI.Chat.ChatCompletionCreateParams = {
-			model: this.options.openAiModelId ?? "",
-			messages: openAiMessages,
-			temperature: 0,
-			stream: true,
-		}
-		if (this.options.includeMaxTokens) {
-			requestOptions.max_tokens = modelInfo.maxTokens
-		}
+		const modelId = this.options.openAiModelId ?? ""
 
-		if (this.options.includeStreamOptions ?? true) {
-			requestOptions.stream_options = { include_usage: true }
-		}
+		if (this.options.openAiStreamingEnabled ?? true) {
+			const systemMessage: OpenAI.Chat.ChatCompletionSystemMessageParam = {
+				role: "system",
+				content: systemPrompt
+			}
+			const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = {
+				model: modelId,
+				temperature: 0,
+				messages: [systemMessage, ...convertToOpenAiMessages(messages)],
+				stream: true as const,
+				stream_options: { include_usage: true },
+			}
+			if (this.options.includeMaxTokens) {
+				requestOptions.max_tokens = modelInfo.maxTokens
+			}
 
-		const stream = await this.client.chat.completions.create(requestOptions)
-		for await (const chunk of stream) {
-			const delta = chunk.choices[0]?.delta
-			if (delta?.content) {
-				yield {
-					type: "text",
-					text: delta.content,
+			const stream = await this.client.chat.completions.create(requestOptions)
+
+			for await (const chunk of stream) {
+				const delta = chunk.choices[0]?.delta
+				if (delta?.content) {
+					yield {
+						type: "text",
+						text: delta.content,
+					}
+				}
+				if (chunk.usage) {
+					yield {
+						type: "usage",
+						inputTokens: chunk.usage.prompt_tokens || 0,
+						outputTokens: chunk.usage.completion_tokens || 0,
+					}
 				}
 			}
-			if (chunk.usage) {
-				yield {
-					type: "usage",
-					inputTokens: chunk.usage.prompt_tokens || 0,
-					outputTokens: chunk.usage.completion_tokens || 0,
-				}
+		} else {
+			// o1 for instance doesnt support streaming, non-1 temp, or system prompt
+			const systemMessage: OpenAI.Chat.ChatCompletionUserMessageParam = {
+				role: "user",
+				content: systemPrompt
+			}
+			const requestOptions: OpenAI.Chat.Completions.ChatCompletionCreateParamsNonStreaming = {
+				model: modelId,
+				messages: [systemMessage, ...convertToOpenAiMessages(messages)],
+			}
+			const response = await this.client.chat.completions.create(requestOptions)
+			
+			yield {
+				type: "text",
+				text: response.choices[0]?.message.content || "",
+			}
+			yield {
+				type: "usage",
+				inputTokens: response.usage?.prompt_tokens || 0,
+				outputTokens: response.usage?.completion_tokens || 0,
 			}
 		}
 	}