Initial work on Vision support in text conversations and Image Editing

This is a huge patch which does some major refactoring like:

- renaming "Image Generation" to "Image Creation" in most places,
  to better match its new command (`!bai image create`)

- relocating image creation command (`!bai image` -> `!bai image create`),
  so it wouldn't conflict with the new image editing command (`!bai image edit`)

- introducing a new image editing command (`!bai image edit`), which
  is meant to work only with the OpenAI provider, but doesn't fully work yet
  due to https://github.com/64bit/async-openai/issues/364, though a next patch will fix it

- adding support for reading images off of Matrix conversations and forwarding them to
  text conversations. Works for OpenAI, but not for Anthropic yet
  (requires custom patches) and not for OpenAI-Compat (no support for
  images there)

- relocating some utils around (base64, mime)

src/agent/provider/openai/controller.rs

@@ -5,7 +5,8 @@ use async_openai::{
     config::OpenAIConfig,
     types::{
         ChatCompletionRequestMessage, CreateChatCompletionRequestArgs, CreateImageRequestArgs,
-        CreateSpeechRequestArgs, CreateTranscriptionRequestArgs,
+        CreateSpeechRequestArgs, CreateTranscriptionRequestArgs, CreateImageEditRequestArgs,
+        ImageInput, ImageModel, DallE2ImageSize, ImageResponseFormat, Image,
     },
 };
 
@@ -14,24 +15,22 @@ use crate::{
     agent::{
         AgentPurpose,
         provider::{
-            entity::{ImageGenerationResult, PingResult, TextToSpeechParams, TextToSpeechResult},
+            entity::{ImageGenerationResult, ImageEditResult, ImageSource, PingResult, TextToSpeechParams, TextToSpeechResult},
             openai::utils::convert_string_to_enum,
         },
     },
     strings,
 };
 use crate::{
-    agent::{
-        provider::{
-            ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
-            entity::{TextGenerationParams, TextGenerationResult},
-        },
-        utils::base64_decode,
+    agent::provider::{
+        ImageGenerationParams, ImageEditParams, SpeechToTextParams, SpeechToTextResult,
+        entity::{TextGenerationParams, TextGenerationResult},
     },
     conversation::llm::{
         Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
-        shorten_messages_list_to_context_size,
+        MessageContent as LLMMessageContent, shorten_messages_list_to_context_size,
     },
+    utils::base64::base64_decode,
 };
 
 use super::config::Config;
@@ -64,7 +63,7 @@ impl ControllerTrait for Controller {
 
         let messages = vec![LLMMessage {
             author: LLMAuthor::User,
-            message_text: "Hello!".to_string(),
+            content: LLMMessageContent::Text("Hello!".to_string()),
             timestamp: chrono::Utc::now(),
         }];
 
@@ -101,7 +100,7 @@ impl ControllerTrait for Controller {
         } else {
             Some(LLMMessage {
                 author: LLMAuthor::Prompt,
-                message_text: prompt_text,
+                content: LLMMessageContent::Text(prompt_text),
                 timestamp: chrono::Utc::now(),
             })
         };
@@ -290,11 +289,13 @@ impl ControllerTrait for Controller {
             .unwrap_or(image_generation_config.size);
 
         let response_format = match model.clone() {
-            async_openai::types::ImageModel::DallE2 => Some(async_openai::types::ImageResponseFormat::B64Json),
-            async_openai::types::ImageModel::DallE3 => Some(async_openai::types::ImageResponseFormat::B64Json),
-            async_openai::types::ImageModel::Other(model_str) => match model_str.as_str() {
+            ImageModel::DallE2 => Some(ImageResponseFormat::B64Json),
+            ImageModel::DallE3 => Some(ImageResponseFormat::B64Json),
+            ImageModel::Other(model_str) => match model_str.as_str() {
+                // gpt-image-1 only outputs base64 and we don't need to specify the response format.
+                // In fact, specifying the response format results in an error.
                 OPENAI_IMAGE_MODEL_GPT_IMAGE_1 => None,
-                _ => Some(async_openai::types::ImageResponseFormat::B64Json),
+                _ => Some(ImageResponseFormat::B64Json),
             },
         };
 
@@ -355,6 +356,96 @@ impl ControllerTrait for Controller {
         ))
     }
 
+    async fn create_image_edit(
+        &self,
+        prompt: &str,
+        images: Vec<ImageSource>,
+        _params: ImageEditParams,
+    ) -> anyhow::Result<ImageEditResult> {
+        let Some(image_generation_config) = &self.config.image_generation else {
+            return Err(anyhow::anyhow!(
+                strings::agent::no_configuration_for_purpose_so_cannot_be_used(
+                    &AgentPurpose::ImageGeneration
+                ),
+            ));
+        };
+
+        let Some(first_image) = images.into_iter().next() else {
+            return Err(anyhow::anyhow!("No image sources provided"));
+        };
+
+        let image_input: ImageInput = first_image.into();
+
+        let dalle2_size = match image_generation_config.size {
+            async_openai::types::ImageSize::S256x256 => Some(DallE2ImageSize::S256x256),
+            async_openai::types::ImageSize::S512x512 => Some(DallE2ImageSize::S512x512),
+            async_openai::types::ImageSize::S1024x1024 => Some(DallE2ImageSize::S1024x1024),
+            _ => None,
+        };
+
+        let model = image_generation_config
+            .model_id_as_openai_image_model()
+            .map_err(|err| anyhow::anyhow!(err))?;
+
+        let response_format = match model.clone() {
+            async_openai::types::ImageModel::DallE2 => Some(async_openai::types::ImageResponseFormat::B64Json),
+            async_openai::types::ImageModel::DallE3 => Some(async_openai::types::ImageResponseFormat::B64Json),
+            async_openai::types::ImageModel::Other(model_str) => match model_str.as_str() {
+                OPENAI_IMAGE_MODEL_GPT_IMAGE_1 => None,
+                _ => Some(async_openai::types::ImageResponseFormat::B64Json),
+            },
+        };
+
+        let mut request_builder = CreateImageEditRequestArgs::default();
+
+        request_builder
+            .image(image_input)
+            .prompt(prompt.to_owned())
+            .model(model);
+
+        if let Some(size) = dalle2_size {
+            request_builder.size(size);
+        }
+
+        if let Some(response_format) = response_format {
+            request_builder.response_format(response_format);
+        }
+
+        let request = request_builder.build()
+            .map_err(|e| anyhow::anyhow!("Failed to build CreateImageEditRequest: {}", e))?;
+
+        tracing::trace!(
+            model = format!("{:?}", request.model),
+            size = format!("{:?}", request.size),
+            response_format = format!("{:?}", request.response_format),
+            "Sending OpenAI image edit API request"
+        );
+
+        let response = self.client.images().create_edit(request).await?;
+
+        if let Some(image_data) = response.data.into_iter().next() {
+            match image_data.deref() {
+                Image::B64Json { b64_json, .. } => {
+                    let bytes = base64_decode(b64_json)?;
+                    return Ok(ImageEditResult {
+                        bytes,
+                        mime_type: mxlink::mime::IMAGE_PNG,
+                    });
+                }
+                Image::Url { url, .. } => {
+                    tracing::warn!(?url, "Received URL instead of B64Json for image edit");
+                    return Err(anyhow::anyhow!(
+                        "Unexpected image type (URL) when B64Json was requested"
+                    ));
+                }
+            }
+        }
+
+        Err(anyhow::anyhow!(
+            "The OpenAI image edit API returned no images"
+        ))
+    }
+
     async fn text_to_speech(
         &self,
         input: &str,

src/agent/provider/openai/utils.rs

@@ -1,9 +1,9 @@
 use async_openai::types::{
-    ChatCompletionRequestAssistantMessageArgs, ChatCompletionRequestMessage,
-    ChatCompletionRequestSystemMessageArgs, ChatCompletionRequestUserMessageArgs,
+    ChatCompletionRequestAssistantMessageArgs, ChatCompletionRequestMessage, ChatCompletionRequestMessageContentPartImage, ChatCompletionRequestSystemMessageArgs, ChatCompletionRequestUserMessageArgs, ChatCompletionRequestUserMessageContent, ChatCompletionRequestUserMessageContentPart, ImageUrlArgs
 };
 
-use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage};
+use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent};
+use crate::utils::base64::base64_encode;
 
 pub fn convert_llm_messages_to_openai_messages(
     conversation_messages: Vec<LLMMessage>,
@@ -12,29 +12,65 @@ pub fn convert_llm_messages_to_openai_messages(
         Vec::with_capacity(conversation_messages.len());
 
     for message in conversation_messages {
-        openai_conversation_messages.push(convert_llm_message_to_openai_message(message));
+        let openai_message = convert_llm_message_to_openai_message(message);
+        if let Some(openai_message) = openai_message {
+            openai_conversation_messages.push(openai_message);
+        }
     }
 
     openai_conversation_messages
 }
 
-fn convert_llm_message_to_openai_message(llm_message: LLMMessage) -> ChatCompletionRequestMessage {
-    match llm_message.author {
-        LLMAuthor::Prompt => ChatCompletionRequestSystemMessageArgs::default()
-            .content(llm_message.message_text)
-            .build()
-            .expect("Failed building OpenAI system message")
-            .into(),
-        LLMAuthor::Assistant => ChatCompletionRequestAssistantMessageArgs::default()
-            .content(llm_message.message_text)
-            .build()
-            .expect("Failed building OpenAI assistant message")
-            .into(),
-        LLMAuthor::User => ChatCompletionRequestUserMessageArgs::default()
-            .content(llm_message.message_text)
-            .build()
-            .expect("Failed building OpenAI user message")
-            .into(),
+fn convert_llm_message_to_openai_message(llm_message: LLMMessage) -> Option<ChatCompletionRequestMessage> {
+    match &llm_message.content {
+        LLMMessageContent::Text(text) => {
+            Some(match llm_message.author {
+                LLMAuthor::Prompt => ChatCompletionRequestSystemMessageArgs::default()
+                    .content(text.clone())
+                    .build()
+                    .expect("Failed building OpenAI system message")
+                    .into(),
+                LLMAuthor::Assistant => ChatCompletionRequestAssistantMessageArgs::default()
+                    .content(text.clone())
+                    .build()
+                    .expect("Failed building OpenAI assistant message")
+                    .into(),
+                LLMAuthor::User => ChatCompletionRequestUserMessageArgs::default()
+                    .content(text.clone())
+                    .build()
+                    .expect("Failed building OpenAI user message")
+                    .into(),
+            })
+        }
+        LLMMessageContent::Image(image_details) => {
+            let image_url = format!("data:{};base64,{}", image_details.mime, base64_encode(&image_details.data));
+
+            let part = ChatCompletionRequestUserMessageContentPart::ImageUrl(
+                ChatCompletionRequestMessageContentPartImage{
+                    image_url: ImageUrlArgs::default()
+                        .url(image_url)
+                        .build()
+                        .expect("Failed building OpenAI image url")
+                }
+            );
+
+            let message_content = ChatCompletionRequestUserMessageContent::Array(vec![part]);
+
+            match llm_message.author {
+                LLMAuthor::User => Some(ChatCompletionRequestUserMessageArgs::default()
+                    .content(message_content)
+                    .build()
+                    .expect("Failed building OpenAI user message")
+                    .into()),
+                _ => {
+                    tracing::warn!(
+                        "OpenAI API does not support image content for messages authored by {:?}. This message part will be skipped.",
+                        llm_message.author
+                    );
+                    None
+                }
+            }
+        }
     }
 }