From ec93f1ee2aa22302922a399c5e37e646a976c1ef Mon Sep 17 00:00:00 2001 From: Layla Date: Fri, 23 Jan 2026 22:05:53 +0100 Subject: [PATCH] Implement OpenAI's response API and add support for built-in tools (web search & code interpreter). --- Cargo.toml | 2 +- docs/sample-provider-configs/openai.yml | 4 + src/agent/provider/openai/config.rs | 13 +++ src/agent/provider/openai/controller.rs | 65 ++++++++----- src/agent/provider/openai/utils.rs | 106 +++++++-------------- src/agent/provider/openai_compat/config.rs | 1 + 6 files changed, 93 insertions(+), 98 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 01a1c86..826a6b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ path = "src/lib.rs" [dependencies] anthropic = { git = "https://github.com/etkecc/anthropic-rs.git", branch = "fix-content-block-image" } anyhow = "1.0.*" -async-openai = { version = "0.32.3", features = ["audio", "chat-completion", "image"] } +async-openai = { version = "0.32.3", features = ["audio", "chat-completion", "image", "responses"] } base64 = "0.22.*" chrono = { version = "0.4.*", default-features = false, features = ["std", "now"] } # We'd rather not depend on this, but we cannot use the ruma-events EventContent macro without it. diff --git a/docs/sample-provider-configs/openai.yml b/docs/sample-provider-configs/openai.yml index e5186a3..b32afbc 100644 --- a/docs/sample-provider-configs/openai.yml +++ b/docs/sample-provider-configs/openai.yml @@ -9,6 +9,10 @@ text_generation: max_response_tokens: null max_completion_tokens: 128000 max_context_tokens: 400000 + # Built-in tools + tools: + web_search: false + code_interpreter: false speech_to_text: model_id: whisper-1 text_to_speech: diff --git a/src/agent/provider/openai/config.rs b/src/agent/provider/openai/config.rs index f055868..ed511f0 100644 --- a/src/agent/provider/openai/config.rs +++ b/src/agent/provider/openai/config.rs @@ -64,6 +64,9 @@ pub struct TextGenerationConfig { #[serde(default)] pub max_context_tokens: u32, + + #[serde(default)] + pub tools: ToolsConfig, } impl Default for TextGenerationConfig { @@ -75,6 +78,7 @@ impl Default for TextGenerationConfig { max_response_tokens: None, max_completion_tokens: Some(128_000), max_context_tokens: 400_000, + tools: ToolsConfig::default(), } } } @@ -83,6 +87,15 @@ fn default_text_model_id() -> String { "gpt-5.2".to_owned() } +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct ToolsConfig { + #[serde(default)] + pub web_search: bool, + + #[serde(default)] + pub code_interpreter: bool, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SpeechToTextConfig { #[serde(default = "default_speech_to_text_model_id")] diff --git a/src/agent/provider/openai/controller.rs b/src/agent/provider/openai/controller.rs index ea21db7..8a2ea32 100644 --- a/src/agent/provider/openai/controller.rs +++ b/src/agent/provider/openai/controller.rs @@ -5,7 +5,10 @@ use async_openai::{ config::OpenAIConfig, types::{ audio::{AudioInput, CreateSpeechRequestArgs, CreateTranscriptionRequestArgs}, - chat::{ChatCompletionRequestMessage, CreateChatCompletionRequestArgs}, + responses::{ + CodeInterpreterContainerAuto, CodeInterpreterTool, CodeInterpreterToolContainer, + CreateResponseArgs, OutputItem, OutputMessageContent, Tool, WebSearchTool, + }, images::{ CreateImageEditRequestArgs, CreateImageRequestArgs, Image, ImageInput, ImageModel, ImageResponseFormat, @@ -129,28 +132,44 @@ impl ControllerTrait for Controller { conversation_messages.insert(0, prompt_message); } - let openai_conversation_messages: Vec = - super::utils::convert_llm_messages_to_openai_messages(conversation_messages); + let input = super::utils::convert_llm_messages_to_openai_response_input(conversation_messages); - let messages_count = openai_conversation_messages.len(); + let messages_count = match &input { + async_openai::types::responses::InputParam::Items(items) => items.len(), + _ => 1, + }; let temperature = params .temperature_override .unwrap_or(text_generation_config.temperature); - let mut request_builder = CreateChatCompletionRequestArgs::default(); + let mut request_builder = CreateResponseArgs::default(); request_builder .model(&text_generation_config.model_id) .temperature(temperature) - .messages(openai_conversation_messages); + .input(input); - if let Some(max_response_tokens) = text_generation_config.max_response_tokens { - request_builder.max_tokens(max_response_tokens); + let mut tools = Vec::new(); + if text_generation_config.tools.web_search { + tools.push(Tool::WebSearch(WebSearchTool::default())); + } + if text_generation_config.tools.code_interpreter { + tools.push(Tool::CodeInterpreter(CodeInterpreterTool { + container: CodeInterpreterToolContainer::Auto( + CodeInterpreterContainerAuto::default(), + ), + })); } - if let Some(max_completion_tokens) = text_generation_config.max_completion_tokens { - request_builder.max_completion_tokens(max_completion_tokens); + if !tools.is_empty() { + request_builder.tools(tools); + } + + if let Some(max_response_tokens) = text_generation_config.max_response_tokens { + request_builder.max_output_tokens(max_response_tokens); + } else if let Some(max_completion_tokens) = text_generation_config.max_completion_tokens { + request_builder.max_output_tokens(max_completion_tokens); } let request = request_builder.build()?; @@ -160,33 +179,31 @@ impl ControllerTrait for Controller { model = format!("{:?}", request.model), ?messages_count, request = request_as_json, - "Sending OpenAI chat completion API request" + "Sending OpenAI response API request" ); } - let response = self.client.chat().create(request).await?; + let response = self.client.responses().create(request).await?; tracing::trace!( ?response, - "Got response from the OpenAI chat completion API" + "Got response from the OpenAI response API" ); - // We only request 1 result, so there should only be 1 choice. - if let Some(choice) = response.choices.into_iter().next() { - match choice.message.content { - Some(text) => { - return Ok(TextGenerationResult { text }); - } - None => { - return Err(anyhow::anyhow!( - "No content was found in the response choice from the OpenAI chat completion API" - )); + for item in response.output { + if let OutputItem::Message(message) = item { + for content in message.content { + if let OutputMessageContent::OutputText(text_content) = content { + return Ok(TextGenerationResult { + text: text_content.text, + }); + } } } } Err(anyhow::anyhow!( - "No response messages choices were returned from the OpenAI chat completion API" + "No response messages choices were returned from the OpenAI response API" )) } diff --git a/src/agent/provider/openai/utils.rs b/src/agent/provider/openai/utils.rs index 966bf90..2baa6ac 100644 --- a/src/agent/provider/openai/utils.rs +++ b/src/agent/provider/openai/utils.rs @@ -1,11 +1,7 @@ use async_openai::types::{ - chat::{ - ChatCompletionRequestAssistantMessageArgs, ChatCompletionRequestMessage, - ChatCompletionRequestMessageContentPartImage, - ChatCompletionRequestSystemMessageArgs, - ChatCompletionRequestUserMessageArgs, ChatCompletionRequestUserMessageContent, - ChatCompletionRequestUserMessageContentPart, - ImageUrlArgs, + responses::{ + EasyInputContent, EasyInputMessage, ImageDetail, InputContent, InputImageContent, + InputItem, InputParam, MessageType, Role, }, }; @@ -14,79 +10,43 @@ use crate::conversation::llm::{ }; use crate::utils::base64::base64_encode; -pub fn convert_llm_messages_to_openai_messages( +pub fn convert_llm_messages_to_openai_response_input( conversation_messages: Vec, -) -> Vec { - let mut openai_conversation_messages: Vec = - Vec::with_capacity(conversation_messages.len()); +) -> InputParam { + let mut items = Vec::with_capacity(conversation_messages.len()); for message in conversation_messages { - let openai_message = convert_llm_message_to_openai_message(message); - if let Some(openai_message) = openai_message { - openai_conversation_messages.push(openai_message); - } - } + let role = match message.author { + LLMAuthor::Prompt => Role::System, + LLMAuthor::Assistant => Role::Assistant, + LLMAuthor::User => Role::User, + }; - openai_conversation_messages -} + let content = match message.content { + LLMMessageContent::Text(text) => EasyInputContent::Text(text), + LLMMessageContent::Image(image_details) => { + let image_url = format!( + "data:{};base64,{}", + image_details.mime, + base64_encode(&image_details.data) + ); -fn convert_llm_message_to_openai_message( - llm_message: LLMMessage, -) -> Option { - match &llm_message.content { - LLMMessageContent::Text(text) => Some(match llm_message.author { - LLMAuthor::Prompt => ChatCompletionRequestSystemMessageArgs::default() - .content(text.clone()) - .build() - .expect("Failed building OpenAI system message") - .into(), - LLMAuthor::Assistant => ChatCompletionRequestAssistantMessageArgs::default() - .content(text.clone()) - .build() - .expect("Failed building OpenAI assistant message") - .into(), - LLMAuthor::User => ChatCompletionRequestUserMessageArgs::default() - .content(text.clone()) - .build() - .expect("Failed building OpenAI user message") - .into(), - }), - LLMMessageContent::Image(image_details) => { - let image_url = format!( - "data:{};base64,{}", - image_details.mime, - base64_encode(&image_details.data) - ); - - let part = ChatCompletionRequestUserMessageContentPart::ImageUrl( - ChatCompletionRequestMessageContentPartImage { - image_url: ImageUrlArgs::default() - .url(image_url) - .build() - .expect("Failed building OpenAI image url"), - }, - ); - - let message_content = ChatCompletionRequestUserMessageContent::Array(vec![part]); - - match llm_message.author { - LLMAuthor::User => Some( - ChatCompletionRequestUserMessageArgs::default() - .content(message_content) - .build() - .expect("Failed building OpenAI user message") - .into(), - ), - _ => { - tracing::warn!( - "OpenAI API does not support image content for messages authored by {:?}. This message part will be skipped.", - llm_message.author - ); - None - } + EasyInputContent::ContentList(vec![InputContent::InputImage(InputImageContent { + image_url: Some(image_url), + detail: ImageDetail::Auto, + file_id: None, + })]) } - } + }; + + items.push(InputItem::EasyMessage(EasyInputMessage { + r#type: MessageType::Message, + role, + content, + })); } + + InputParam::Items(items) } pub(super) fn convert_string_to_enum(value: &str) -> Result diff --git a/src/agent/provider/openai_compat/config.rs b/src/agent/provider/openai_compat/config.rs index 3315fa8..22b94e7 100644 --- a/src/agent/provider/openai_compat/config.rs +++ b/src/agent/provider/openai_compat/config.rs @@ -95,6 +95,7 @@ impl TryInto for TextGenerationConfig { max_response_tokens: self.max_response_tokens, max_completion_tokens: None, max_context_tokens: self.max_context_tokens, + tools: Default::default(), }) } }