Initial work on Vision support in text conversations and Image Editing

This is a huge patch which does some major refactoring like:

- renaming "Image Generation" to "Image Creation" in most places,
  to better match its new command (`!bai image create`)

- relocating image creation command (`!bai image` -> `!bai image create`),
  so it wouldn't conflict with the new image editing command (`!bai image edit`)

- introducing a new image editing command (`!bai image edit`), which
  is meant to work only with the OpenAI provider, but doesn't fully work yet
  due to https://github.com/64bit/async-openai/issues/364, though a next patch will fix it

- adding support for reading images off of Matrix conversations and forwarding them to
  text conversations. Works for OpenAI, but not for Anthropic yet
  (requires custom patches) and not for OpenAI-Compat (no support for
  images there)

- relocating some utils around (base64, mime)
This commit is contained in:
Slavi Pantaleev
2025-05-10 09:18:01 +03:00
parent e0dcc39a72
commit 8f86289373
57 changed files with 1074 additions and 335 deletions

View File

@@ -7,12 +7,14 @@ use anthropic::types::ContentBlock;
use super::super::ControllerTrait;
use crate::agent::AgentPurpose;
use crate::agent::provider::entity::{
ImageGenerationResult, PingResult, TextGenerationParams, TextGenerationResult,
TextToSpeechParams, TextToSpeechResult,
ImageGenerationResult, ImageEditResult, ImageSource, PingResult, TextGenerationParams,
TextGenerationResult, TextToSpeechParams, TextToSpeechResult,
};
use crate::agent::provider::{
ImageGenerationParams, ImageEditParams, SpeechToTextParams, SpeechToTextResult,
};
use crate::agent::provider::{ImageGenerationParams, SpeechToTextParams, SpeechToTextResult};
use crate::conversation::llm::{
Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage, MessageContent as LLMMessageContent,
shorten_messages_list_to_context_size,
};
use crate::strings;
@@ -69,7 +71,7 @@ impl ControllerTrait for Controller {
let messages = vec![LLMMessage {
author: LLMAuthor::User,
message_text: "Hello!".to_string(),
content: LLMMessageContent::Text("Hello!".to_string()),
timestamp: chrono::Utc::now(),
}];
@@ -106,7 +108,7 @@ impl ControllerTrait for Controller {
} else {
Some(LLMMessage {
author: LLMAuthor::Prompt,
message_text: prompt_text,
content: LLMMessageContent::Text(prompt_text),
timestamp: chrono::Utc::now(),
})
};
@@ -145,7 +147,9 @@ impl ControllerTrait for Controller {
.unwrap_or(text_generation_config.temperature);
if let Some(prompt_message) = prompt_message {
request.system = prompt_message.message_text;
if let LLMMessageContent::Text(text) = &prompt_message.content {
request.system = text.clone();
}
}
request.model = text_generation_config.model_id.clone();
@@ -213,6 +217,15 @@ impl ControllerTrait for Controller {
Err(anyhow::anyhow!("Image generation not supported"))
}
async fn create_image_edit(
&self,
_prompt: &str,
_images: Vec<ImageSource>,
_params: ImageEditParams,
) -> anyhow::Result<ImageEditResult> {
Err(anyhow::anyhow!("Image editing is not supported"))
}
async fn text_to_speech(
&self,
_input: &str,

View File

@@ -1,6 +1,6 @@
use anthropic::types::{ContentBlock, Message, MessagesRequest, MessagesRequestBuilder, Role};
use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage};
use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent};
pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) -> MessagesRequest {
let mut messages = vec![];
@@ -14,13 +14,28 @@ pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) ->
}
};
let content = vec![ContentBlock::Text {
text: message.message_text,
}];
let content = match &message.content {
LLMMessageContent::Text(text) => Some(vec![ContentBlock::Text { text: text.clone() }]),
LLMMessageContent::Image(_image_details) => {
// This cannot be implemented yet, because the Anthropic library does not support it.
// The code below requires the library to be forked and changed a bit.
// vec![ContentBlock::Image {
// source: ImageSource {
// r#type: "base64".to_string(),
// media_type: mime_type.to_string(),
// data: crate::utils::base64::base64_encode(image_details.data),
// },
// }]
tracing::warn!("Image content is not supported by the Anthropic library yet. Skipping it.");
None
}
};
let message = Message { role, content };
if let Some(content) = content {
let message = Message { role, content };
messages.push(message);
messages.push(message);
}
}
MessagesRequestBuilder::default()