Initial work on Vision support in text conversations and Image Editing
This is a huge patch which does some major refactoring like: - renaming "Image Generation" to "Image Creation" in most places, to better match its new command (`!bai image create`) - relocating image creation command (`!bai image` -> `!bai image create`), so it wouldn't conflict with the new image editing command (`!bai image edit`) - introducing a new image editing command (`!bai image edit`), which is meant to work only with the OpenAI provider, but doesn't fully work yet due to https://github.com/64bit/async-openai/issues/364, though a next patch will fix it - adding support for reading images off of Matrix conversations and forwarding them to text conversations. Works for OpenAI, but not for Anthropic yet (requires custom patches) and not for OpenAI-Compat (no support for images there) - relocating some utils around (base64, mime)
This commit is contained in:
@@ -7,12 +7,14 @@ use anthropic::types::ContentBlock;
|
||||
use super::super::ControllerTrait;
|
||||
use crate::agent::AgentPurpose;
|
||||
use crate::agent::provider::entity::{
|
||||
ImageGenerationResult, PingResult, TextGenerationParams, TextGenerationResult,
|
||||
TextToSpeechParams, TextToSpeechResult,
|
||||
ImageGenerationResult, ImageEditResult, ImageSource, PingResult, TextGenerationParams,
|
||||
TextGenerationResult, TextToSpeechParams, TextToSpeechResult,
|
||||
};
|
||||
use crate::agent::provider::{
|
||||
ImageGenerationParams, ImageEditParams, SpeechToTextParams, SpeechToTextResult,
|
||||
};
|
||||
use crate::agent::provider::{ImageGenerationParams, SpeechToTextParams, SpeechToTextResult};
|
||||
use crate::conversation::llm::{
|
||||
Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
|
||||
Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage, MessageContent as LLMMessageContent,
|
||||
shorten_messages_list_to_context_size,
|
||||
};
|
||||
use crate::strings;
|
||||
@@ -69,7 +71,7 @@ impl ControllerTrait for Controller {
|
||||
|
||||
let messages = vec![LLMMessage {
|
||||
author: LLMAuthor::User,
|
||||
message_text: "Hello!".to_string(),
|
||||
content: LLMMessageContent::Text("Hello!".to_string()),
|
||||
timestamp: chrono::Utc::now(),
|
||||
}];
|
||||
|
||||
@@ -106,7 +108,7 @@ impl ControllerTrait for Controller {
|
||||
} else {
|
||||
Some(LLMMessage {
|
||||
author: LLMAuthor::Prompt,
|
||||
message_text: prompt_text,
|
||||
content: LLMMessageContent::Text(prompt_text),
|
||||
timestamp: chrono::Utc::now(),
|
||||
})
|
||||
};
|
||||
@@ -145,7 +147,9 @@ impl ControllerTrait for Controller {
|
||||
.unwrap_or(text_generation_config.temperature);
|
||||
|
||||
if let Some(prompt_message) = prompt_message {
|
||||
request.system = prompt_message.message_text;
|
||||
if let LLMMessageContent::Text(text) = &prompt_message.content {
|
||||
request.system = text.clone();
|
||||
}
|
||||
}
|
||||
|
||||
request.model = text_generation_config.model_id.clone();
|
||||
@@ -213,6 +217,15 @@ impl ControllerTrait for Controller {
|
||||
Err(anyhow::anyhow!("Image generation not supported"))
|
||||
}
|
||||
|
||||
async fn create_image_edit(
|
||||
&self,
|
||||
_prompt: &str,
|
||||
_images: Vec<ImageSource>,
|
||||
_params: ImageEditParams,
|
||||
) -> anyhow::Result<ImageEditResult> {
|
||||
Err(anyhow::anyhow!("Image editing is not supported"))
|
||||
}
|
||||
|
||||
async fn text_to_speech(
|
||||
&self,
|
||||
_input: &str,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use anthropic::types::{ContentBlock, Message, MessagesRequest, MessagesRequestBuilder, Role};
|
||||
|
||||
use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage};
|
||||
use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent};
|
||||
|
||||
pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) -> MessagesRequest {
|
||||
let mut messages = vec![];
|
||||
@@ -14,13 +14,28 @@ pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) ->
|
||||
}
|
||||
};
|
||||
|
||||
let content = vec![ContentBlock::Text {
|
||||
text: message.message_text,
|
||||
}];
|
||||
let content = match &message.content {
|
||||
LLMMessageContent::Text(text) => Some(vec![ContentBlock::Text { text: text.clone() }]),
|
||||
LLMMessageContent::Image(_image_details) => {
|
||||
// This cannot be implemented yet, because the Anthropic library does not support it.
|
||||
// The code below requires the library to be forked and changed a bit.
|
||||
// vec![ContentBlock::Image {
|
||||
// source: ImageSource {
|
||||
// r#type: "base64".to_string(),
|
||||
// media_type: mime_type.to_string(),
|
||||
// data: crate::utils::base64::base64_encode(image_details.data),
|
||||
// },
|
||||
// }]
|
||||
tracing::warn!("Image content is not supported by the Anthropic library yet. Skipping it.");
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
let message = Message { role, content };
|
||||
if let Some(content) = content {
|
||||
let message = Message { role, content };
|
||||
|
||||
messages.push(message);
|
||||
messages.push(message);
|
||||
}
|
||||
}
|
||||
|
||||
MessagesRequestBuilder::default()
|
||||
|
||||
Reference in New Issue
Block a user