From 8eb70f0f2ce7440ceca169d1619e6719e1ad7198 Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Sun, 30 Nov 2025 10:31:06 +0200 Subject: [PATCH] Upgrade `async-openai` from our own etkecc fork to upstream's 0.31.1 Switches `async-openai` from our own etkecc fork (0.28.1-patched) to the official crates.io version 0.31.1. We adapt to async-openai's types reorganization and making use of crate features to only enable what we need. --- Cargo.lock | 12 ++- Cargo.toml | 2 +- src/agent/provider/entity/image.rs | 8 +- src/agent/provider/openai/config.rs | 40 +++---- src/agent/provider/openai/controller.rs | 115 +++++++++++---------- src/agent/provider/openai/utils.rs | 12 ++- src/agent/provider/openai_compat/config.rs | 12 +-- 7 files changed, 109 insertions(+), 92 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19ea769..2d0c7dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -191,8 +191,9 @@ dependencies = [ [[package]] name = "async-openai" -version = "0.28.1" -source = "git+https://github.com/etkecc/async-openai?branch=async-openai-v0.28.1-patched#856953c2d4485342df625fd0525363362075e8a8" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cc4c602409022b854d89332fae01a6c69a4122fdd0ec66a071b9b5c3a87750b" dependencies = [ "async-openai-macros", "backoff", @@ -201,23 +202,26 @@ dependencies = [ "derive_builder 0.20.2", "eventsource-stream", "futures", - "rand 0.8.5", + "rand 0.9.2", "reqwest 0.12.24", "reqwest-eventsource 0.6.0", "secrecy", "serde", "serde_json", + "serde_urlencoded", "thiserror 2.0.17", "tokio", "tokio-stream", "tokio-util", "tracing", + "url", ] [[package]] name = "async-openai-macros" version = "0.1.0" -source = "git+https://github.com/etkecc/async-openai?branch=async-openai-v0.28.1-patched#856953c2d4485342df625fd0525363362075e8a8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0289cba6d5143bfe8251d57b4a8cac036adf158525a76533a7082ba65ec76398" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 80e4ac7..b586441 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ path = "src/lib.rs" [dependencies] anthropic = { git = "https://github.com/etkecc/anthropic-rs.git", branch = "fix-content-block-image" } anyhow = "1.0.*" -async-openai = { git = "https://github.com/etkecc/async-openai", branch = "async-openai-v0.28.1-patched" } +async-openai = { version = "0.31.1", features = ["audio", "chat-completion", "image"] } base64 = "0.22.*" chrono = { version = "0.4.*", default-features = false, features = ["std", "now"] } # We'd rather not depend on this, but we cannot use the ruma-events EventContent macro without it. diff --git a/src/agent/provider/entity/image.rs b/src/agent/provider/entity/image.rs index df95a7f..7e7c49a 100644 --- a/src/agent/provider/entity/image.rs +++ b/src/agent/provider/entity/image.rs @@ -56,12 +56,8 @@ impl ImageSource { } } -impl From for async_openai::types::ImageInput { +impl From for async_openai::types::images::ImageInput { fn from(value: ImageSource) -> Self { - async_openai::types::ImageInput::from_vec_u8( - value.filename, - value.bytes, - value.mime_type.to_string(), - ) + async_openai::types::images::ImageInput::from_vec_u8(value.filename, value.bytes) } } diff --git a/src/agent/provider/openai/config.rs b/src/agent/provider/openai/config.rs index 330a600..e503543 100644 --- a/src/agent/provider/openai/config.rs +++ b/src/agent/provider/openai/config.rs @@ -104,16 +104,16 @@ fn default_speech_to_text_model_id() -> String { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TextToSpeechConfig { #[serde(default = "default_text_to_speech_model_id")] - pub model_id: async_openai::types::SpeechModel, + pub model_id: async_openai::types::audio::SpeechModel, #[serde(default = "default_text_to_speech_voice")] - pub voice: async_openai::types::Voice, + pub voice: async_openai::types::audio::Voice, #[serde(default = "default_text_to_speech_speed")] pub speed: f32, #[serde(default = "default_text_to_speech_response_format")] - pub response_format: async_openai::types::SpeechResponseFormat, + pub response_format: async_openai::types::audio::SpeechResponseFormat, } impl Default for TextToSpeechConfig { @@ -127,22 +127,22 @@ impl Default for TextToSpeechConfig { } } -fn default_text_to_speech_model_id() -> async_openai::types::SpeechModel { - async_openai::types::SpeechModel::Tts1Hd +fn default_text_to_speech_model_id() -> async_openai::types::audio::SpeechModel { + async_openai::types::audio::SpeechModel::Tts1Hd } -fn default_text_to_speech_voice() -> async_openai::types::Voice { - async_openai::types::Voice::Onyx +fn default_text_to_speech_voice() -> async_openai::types::audio::Voice { + async_openai::types::audio::Voice::Onyx } fn default_text_to_speech_speed() -> f32 { 1.0 } -fn default_text_to_speech_response_format() -> async_openai::types::SpeechResponseFormat { +fn default_text_to_speech_response_format() -> async_openai::types::audio::SpeechResponseFormat { // The API defaults to mp3, but we prefer Opus because it's smaller. // Our clients should all have support for it. - async_openai::types::SpeechResponseFormat::Opus + async_openai::types::audio::SpeechResponseFormat::Opus } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -150,13 +150,13 @@ pub struct ImageGenerationConfig { pub model_id: String, #[serde(default = "default_image_style")] - pub style: Option, + pub style: Option, #[serde(default = "default_image_size")] - pub size: Option, + pub size: Option, #[serde(default = "default_image_quality")] - pub quality: Option, + pub quality: Option, } impl Default for ImageGenerationConfig { @@ -173,23 +173,25 @@ impl Default for ImageGenerationConfig { impl ImageGenerationConfig { pub fn model_id_as_openai_image_model( &self, - ) -> Result { + ) -> Result { match self.model_id.as_str() { - "dall-e-2" => Ok(async_openai::types::ImageModel::DallE2), - "dall-e-3" => Ok(async_openai::types::ImageModel::DallE3), - other => Ok(async_openai::types::ImageModel::Other(other.to_owned())), + "dall-e-2" => Ok(async_openai::types::images::ImageModel::DallE2), + "dall-e-3" => Ok(async_openai::types::images::ImageModel::DallE3), + "gpt-image-1" => Ok(async_openai::types::images::ImageModel::GptImage1), + "gpt-image-1-mini" => Ok(async_openai::types::images::ImageModel::GptImage1Mini), + other => Ok(async_openai::types::images::ImageModel::Other(other.to_owned())), } } } -fn default_image_style() -> Option { +fn default_image_style() -> Option { None } -fn default_image_size() -> Option { +fn default_image_size() -> Option { None } -fn default_image_quality() -> Option { +fn default_image_quality() -> Option { None } diff --git a/src/agent/provider/openai/controller.rs b/src/agent/provider/openai/controller.rs index bd6ee86..b043b0d 100644 --- a/src/agent/provider/openai/controller.rs +++ b/src/agent/provider/openai/controller.rs @@ -4,9 +4,12 @@ use async_openai::{ Client as OpenAIClient, config::OpenAIConfig, types::{ - ChatCompletionRequestMessage, CreateChatCompletionRequestArgs, CreateImageEditRequestArgs, - CreateImageRequestArgs, CreateSpeechRequestArgs, CreateTranscriptionRequestArgs, - DallE2ImageSize, Image, ImageModel, ImageResponseFormat, + audio::{AudioInput, CreateSpeechRequestArgs, CreateTranscriptionRequestArgs}, + chat::{ChatCompletionRequestMessage, CreateChatCompletionRequestArgs}, + images::{ + CreateImageEditRequestArgs, CreateImageRequestArgs, + Image, ImageInput, ImageModel, ImageResponseFormat, + }, }, }; @@ -38,8 +41,6 @@ use crate::{ use super::config::Config; -use super::OPENAI_IMAGE_MODEL_GPT_IMAGE_1; - #[derive(Debug, Clone)] pub struct Controller { config: Config, @@ -209,11 +210,7 @@ impl ControllerTrait for Controller { let request = CreateTranscriptionRequestArgs::default() .model(&speech_to_text_config.model_id) - .file(async_openai::types::AudioInput::from_vec_u8( - filename, - media, - mime_type.to_string(), - )) + .file(AudioInput::from_vec_u8(filename, media)) .language(language.clone()) .build()?; @@ -223,7 +220,7 @@ impl ControllerTrait for Controller { "Sending OpenAI speech-to-text API request" ); - let response = self.client.audio().transcribe(request).await?; + let response = self.client.audio().transcription().create(request).await?; tracing::trace!( ?response, @@ -255,11 +252,12 @@ impl ControllerTrait for Controller { let model = if params.cheaper_model_switching_allowed { // Switch to a cheaper model match original_model { - async_openai::types::ImageModel::DallE2 => async_openai::types::ImageModel::DallE2, - async_openai::types::ImageModel::DallE3 => async_openai::types::ImageModel::DallE2, - async_openai::types::ImageModel::Other(_) => { - async_openai::types::ImageModel::DallE2 + ImageModel::DallE2 => ImageModel::DallE2, + ImageModel::DallE3 => ImageModel::DallE2, + ImageModel::Other(_) => { + ImageModel::DallE2 } + _ => original_model.clone(), } } else { original_model @@ -269,11 +267,24 @@ impl ControllerTrait for Controller { // Switch to a cheaper quality match &image_generation_config.quality { Some(quality) => match quality { - async_openai::types::ImageQuality::Standard => { - Some(async_openai::types::ImageQuality::Standard) + async_openai::types::images::ImageQuality::Standard => { + Some(async_openai::types::images::ImageQuality::Standard) } - async_openai::types::ImageQuality::HD => { - Some(async_openai::types::ImageQuality::Standard) + async_openai::types::images::ImageQuality::HD => { + Some(async_openai::types::images::ImageQuality::Standard) + } + // New quality levels - keep as-is or downgrade to Standard + async_openai::types::images::ImageQuality::High => { + Some(async_openai::types::images::ImageQuality::Standard) + } + async_openai::types::images::ImageQuality::Medium => { + Some(async_openai::types::images::ImageQuality::Medium) + } + async_openai::types::images::ImageQuality::Low => { + Some(async_openai::types::images::ImageQuality::Low) + } + async_openai::types::images::ImageQuality::Auto => { + Some(async_openai::types::images::ImageQuality::Auto) } }, None => None, @@ -284,18 +295,17 @@ impl ControllerTrait for Controller { let size = params .size_override - .map(|s| convert_string_to_enum::(&s).unwrap()) + .map(|s| convert_string_to_enum::(&s).unwrap()) .or(image_generation_config.size); let response_format = match model.clone() { ImageModel::DallE2 => Some(ImageResponseFormat::B64Json), ImageModel::DallE3 => Some(ImageResponseFormat::B64Json), - ImageModel::Other(model_str) => match model_str.as_str() { - // gpt-image-1 only outputs base64 and we don't need to specify the response format. - // In fact, specifying the response format results in an error. - OPENAI_IMAGE_MODEL_GPT_IMAGE_1 => None, - _ => Some(ImageResponseFormat::B64Json), - }, + // gpt-image-1 only outputs base64 and we don't need to specify the response format. + // In fact, specifying the response format results in an error. + ImageModel::GptImage1 => None, + ImageModel::GptImage1Mini => None, + ImageModel::Other(_) => Some(ImageResponseFormat::B64Json), }; let mut request_builder = CreateImageRequestArgs::default(); @@ -329,15 +339,15 @@ impl ControllerTrait for Controller { "Sending OpenAI image generation API request" ); - let response = self.client.images().create(request).await?; + let response = self.client.images().generate(request).await?; if let Some(image) = response.data.into_iter().next() { match image.deref() { - async_openai::types::Image::B64Json { + Image::B64Json { b64_json, revised_prompt, } => { - let bytes = base64_decode(b64_json)?; + let bytes = base64_decode(b64_json.as_ref())?; return Ok(ImageGenerationResult { bytes, @@ -374,15 +384,15 @@ impl ControllerTrait for Controller { return Err(anyhow::anyhow!("No image sources provided")); } - let mut image_inputs = Vec::new(); + let mut image_inputs: Vec = Vec::new(); for image in images { image_inputs.push(image.into()); } let dalle2_size = match image_generation_config.size { - Some(async_openai::types::ImageSize::S256x256) => Some(DallE2ImageSize::S256x256), - Some(async_openai::types::ImageSize::S512x512) => Some(DallE2ImageSize::S512x512), - Some(async_openai::types::ImageSize::S1024x1024) => Some(DallE2ImageSize::S1024x1024), + Some(async_openai::types::images::ImageSize::S256x256) => Some(async_openai::types::images::ImageSize::S256x256), + Some(async_openai::types::images::ImageSize::S512x512) => Some(async_openai::types::images::ImageSize::S512x512), + Some(async_openai::types::images::ImageSize::S1024x1024) => Some(async_openai::types::images::ImageSize::S1024x1024), _ => None, }; @@ -391,16 +401,17 @@ impl ControllerTrait for Controller { .map_err(|err| anyhow::anyhow!(err))?; let response_format = match model.clone() { - async_openai::types::ImageModel::DallE2 => { - Some(async_openai::types::ImageResponseFormat::B64Json) + ImageModel::DallE2 => { + Some(ImageResponseFormat::B64Json) } - async_openai::types::ImageModel::DallE3 => { - Some(async_openai::types::ImageResponseFormat::B64Json) + ImageModel::DallE3 => { + Some(ImageResponseFormat::B64Json) } - async_openai::types::ImageModel::Other(model_str) => match model_str.as_str() { - OPENAI_IMAGE_MODEL_GPT_IMAGE_1 => None, - _ => Some(async_openai::types::ImageResponseFormat::B64Json), - }, + // gpt-image-1 only outputs base64 and we don't need to specify the response format. + // In fact, specifying the response format results in an error. + ImageModel::GptImage1 => None, + ImageModel::GptImage1Mini => None, + ImageModel::Other(_) => Some(ImageResponseFormat::B64Json), }; let mut request_builder = CreateImageEditRequestArgs::default(); @@ -429,12 +440,12 @@ impl ControllerTrait for Controller { "Sending OpenAI image edit API request" ); - let response = self.client.images().create_edit(request).await?; + let response = self.client.images().edit(request).await?; if let Some(image_data) = response.data.into_iter().next() { match image_data.deref() { Image::B64Json { b64_json, .. } => { - let bytes = base64_decode(b64_json)?; + let bytes = base64_decode(b64_json.as_ref())?; return Ok(ImageEditResult { bytes, mime_type: mxlink::mime::IMAGE_PNG, @@ -471,7 +482,7 @@ impl ControllerTrait for Controller { let voice = if let Some(voice_string) = params.voice_override { // This is a hacky way to construct a Voice enum from the string we have. - let voice: serde_json::Result = + let voice: serde_json::Result = serde_json::from_str(&format!("\"{}\"", voice_string)); match voice { Ok(voice) => voice, @@ -511,7 +522,7 @@ impl ControllerTrait for Controller { "Sending OpenAI text-to-speech API request" ); - let result = self.client.audio().speech(request).await?; + let result = self.client.audio().speech().create(request).await?; Ok(TextToSpeechResult { bytes: result.bytes.into(), @@ -570,15 +581,15 @@ impl ControllerTrait for Controller { } fn response_format_to_mime_type( - response_format: &async_openai::types::SpeechResponseFormat, + response_format: &async_openai::types::audio::SpeechResponseFormat, ) -> Option { let content_type = match response_format { - async_openai::types::SpeechResponseFormat::Mp3 => "audio/mp3".to_owned(), - async_openai::types::SpeechResponseFormat::Wav => "audio/wav".to_owned(), - async_openai::types::SpeechResponseFormat::Opus => "audio/ogg".to_owned(), - async_openai::types::SpeechResponseFormat::Aac => "audio/aac".to_owned(), - async_openai::types::SpeechResponseFormat::Flac => "audio/flac".to_owned(), - async_openai::types::SpeechResponseFormat::Pcm => "audio/L8".to_owned(), + async_openai::types::audio::SpeechResponseFormat::Mp3 => "audio/mp3".to_owned(), + async_openai::types::audio::SpeechResponseFormat::Wav => "audio/wav".to_owned(), + async_openai::types::audio::SpeechResponseFormat::Opus => "audio/ogg".to_owned(), + async_openai::types::audio::SpeechResponseFormat::Aac => "audio/aac".to_owned(), + async_openai::types::audio::SpeechResponseFormat::Flac => "audio/flac".to_owned(), + async_openai::types::audio::SpeechResponseFormat::Pcm => "audio/L8".to_owned(), }; match content_type.parse() { diff --git a/src/agent/provider/openai/utils.rs b/src/agent/provider/openai/utils.rs index f477f0a..966bf90 100644 --- a/src/agent/provider/openai/utils.rs +++ b/src/agent/provider/openai/utils.rs @@ -1,8 +1,12 @@ use async_openai::types::{ - ChatCompletionRequestAssistantMessageArgs, ChatCompletionRequestMessage, - ChatCompletionRequestMessageContentPartImage, ChatCompletionRequestSystemMessageArgs, - ChatCompletionRequestUserMessageArgs, ChatCompletionRequestUserMessageContent, - ChatCompletionRequestUserMessageContentPart, ImageUrlArgs, + chat::{ + ChatCompletionRequestAssistantMessageArgs, ChatCompletionRequestMessage, + ChatCompletionRequestMessageContentPartImage, + ChatCompletionRequestSystemMessageArgs, + ChatCompletionRequestUserMessageArgs, ChatCompletionRequestUserMessageContent, + ChatCompletionRequestUserMessageContentPart, + ImageUrlArgs, + }, }; use crate::conversation::llm::{ diff --git a/src/agent/provider/openai_compat/config.rs b/src/agent/provider/openai_compat/config.rs index e4ed370..3315fa8 100644 --- a/src/agent/provider/openai_compat/config.rs +++ b/src/agent/provider/openai_compat/config.rs @@ -161,11 +161,11 @@ impl TryInto for TextToSpeechConfig { type Error = String; fn try_into(self) -> Result { - let model_id = convert_string_to_enum::(&self.model_id)?; + let model_id = convert_string_to_enum::(&self.model_id)?; - let voice = convert_string_to_enum::(&self.voice)?; + let voice = convert_string_to_enum::(&self.voice)?; - let response_format = convert_string_to_enum::( + let response_format = convert_string_to_enum::( &self.response_format, )?; @@ -224,7 +224,7 @@ impl TryInto for ImageGenerationConfig { fn try_into(self) -> Result { let size = if let Some(size) = &self.size { - Some(convert_string_to_enum::( + Some(convert_string_to_enum::( size, )?) } else { @@ -232,7 +232,7 @@ impl TryInto for ImageGenerationConfig { }; let style = if let Some(style) = &self.style { - Some(convert_string_to_enum::( + Some(convert_string_to_enum::( style, )?) } else { @@ -240,7 +240,7 @@ impl TryInto for ImageGenerationConfig { }; let quality = if let Some(quality) = &self.quality { - Some(convert_string_to_enum::( + Some(convert_string_to_enum::( quality, )?) } else {