diff --git a/Cargo.lock b/Cargo.lock index 6a3767f..e22f4a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -228,8 +228,7 @@ dependencies = [ [[package]] name = "async-openai" version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14d76e2f5af19477d6254415acc95ba97c6cc6f3b1e3cb4676b7f0fab8194298" +source = "git+https://github.com/etkecc/async-openai?branch=async-openai-v0.28.1-patched#856953c2d4485342df625fd0525363362075e8a8" dependencies = [ "async-openai-macros", "backoff", @@ -254,8 +253,7 @@ dependencies = [ [[package]] name = "async-openai-macros" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0289cba6d5143bfe8251d57b4a8cac036adf158525a76533a7082ba65ec76398" +source = "git+https://github.com/etkecc/async-openai?branch=async-openai-v0.28.1-patched#856953c2d4485342df625fd0525363362075e8a8" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index d41bd37..4e75e62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ path = "src/lib.rs" [dependencies] anthropic = "=0.0.8" anyhow = "1.0.*" -async-openai = "0.28.*" +async-openai = { git = "https://github.com/etkecc/async-openai", branch = "async-openai-v0.28.1-patched" } base64 = "0.22.*" chrono = { version = "0.4.*", default-features = false, features = ["std", "now"] } # We'd rather not depend on this, but we cannot use the ruma-events EventContent macro without it. diff --git a/src/agent/provider/entity/image.rs b/src/agent/provider/entity/image.rs index f85e367..b524f41 100644 --- a/src/agent/provider/entity/image.rs +++ b/src/agent/provider/entity/image.rs @@ -58,6 +58,7 @@ impl From for async_openai::types::ImageInput { async_openai::types::ImageInput::from_vec_u8( value.filename, value.bytes, + value.mime_type.to_string(), ) } } diff --git a/src/agent/provider/openai/controller.rs b/src/agent/provider/openai/controller.rs index 2dd19e9..e2e1299 100644 --- a/src/agent/provider/openai/controller.rs +++ b/src/agent/provider/openai/controller.rs @@ -6,7 +6,7 @@ use async_openai::{ types::{ ChatCompletionRequestMessage, CreateChatCompletionRequestArgs, CreateImageRequestArgs, CreateSpeechRequestArgs, CreateTranscriptionRequestArgs, CreateImageEditRequestArgs, - ImageInput, ImageModel, DallE2ImageSize, ImageResponseFormat, Image, + ImageModel, DallE2ImageSize, ImageResponseFormat, Image, }, }; @@ -209,6 +209,7 @@ impl ControllerTrait for Controller { .file(async_openai::types::AudioInput::from_vec_u8( filename, media, + mime_type.to_string(), )) .language(language.clone()) .build()?; @@ -368,11 +369,14 @@ impl ControllerTrait for Controller { )); }; - let Some(first_image) = images.into_iter().next() else { + if images.is_empty() { return Err(anyhow::anyhow!("No image sources provided")); - }; + } - let image_input: ImageInput = first_image.into(); + let mut image_inputs = Vec::new(); + for image in images { + image_inputs.push(image.into()); + } let dalle2_size = match image_generation_config.size { async_openai::types::ImageSize::S256x256 => Some(DallE2ImageSize::S256x256), @@ -397,7 +401,7 @@ impl ControllerTrait for Controller { let mut request_builder = CreateImageEditRequestArgs::default(); request_builder - .image(image_input) + .image(image_inputs) .prompt(prompt.to_owned()) .model(model);