diff --git a/docs/configuration/handlers.md b/docs/configuration/handlers.md index 97a9dd6..bd32760 100644 --- a/docs/configuration/handlers.md +++ b/docs/configuration/handlers.md @@ -8,7 +8,7 @@ You can also use **different models within the same room** (e.g. [đŸ’Ŧ text-gene The bot supports the following use-purposes: -- [đŸ’Ŧ text-generation](../features.md#-text-generation): communicating with you via text (though certain models may "see" images as well) +- [đŸ’Ŧ text-generation](../features.md#-text-generation): communicating with you via text (though certain models may also process images and files) - [đŸĻģ speech-to-text](../features.md#-speech-to-text): turning your voice messages into text - [đŸ—Ŗī¸ text-to-speech](../features.md#ī¸-text-to-speech): turning bot or users text messages into voice messages - [🖌ī¸ image-generation](../features.md#image-generation): generating images based on instructions diff --git a/docs/features.md b/docs/features.md index 93df593..9e77f84 100644 --- a/docs/features.md +++ b/docs/features.md @@ -8,7 +8,7 @@ You can also use **different models within the same room** (e.g. [đŸ’Ŧ text-gene The bot supports the following use-purposes: -- [đŸ’Ŧ text-generation](#-text-generation): communicating with you via text (though certain models may "see" images as well) +- [đŸ’Ŧ text-generation](#-text-generation): communicating with you via text (though certain models may also process images and files) - [đŸĻģ speech-to-text](#-speech-to-text): turning your voice messages into text - [đŸ—Ŗī¸ text-to-speech](#%EF%B8%8F-text-to-speech): turning bot or users text messages into voice messages - [🖌ī¸ image-generation](#%EF%B8%8F-image-generation): generating images based on instructions @@ -26,7 +26,7 @@ Text Generation is the bot's ability to **respond to users' messages with text** ![Screenshot of Text Generation - a user sends a message and the bot replies in a new conversation thread](./screenshots/text-generation.webp) -Some models also support vision, so you may be able to mix text and images in the same conversation. +Some models also support vision and document understanding, so you may be able to mix text, images, and files (PDFs, text documents, etc.) in the same conversation. Note that certain providers may not support all file types or may have issues with specific files (e.g. scanned/image-based PDFs). If a file is rejected by the provider, the conversation thread may become unusable — start a new thread to work around this. In multi-user (group) rooms, to avoid disturbing the normal conversation between people, the bot is auto-configured to only respond to messages starting with the command prefix (`!bai`) or direct mentions via the [đŸ’Ŧ Text Generation / 🗟 Prefix Requirement Type](./configuration/text-generation.md#-prefix-requirement-type) setting. diff --git a/docs/usage.md b/docs/usage.md index 50ebc34..f42ac61 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -11,7 +11,7 @@ This is related to the [đŸ’Ŧ Text Generation](./features.md#-text-generation) fe If there's a text-generation handler agent configured, the bot **may** respond to messages sent in the room. -Some models also support vision, so you may be able to mix text and images in the same conversation. +Some models also support vision and document understanding, so you may be able to mix text, images, and files (PDFs, text documents, etc.) in the same conversation. See screenshots of: diff --git a/src/agent/provider/anthropic/utils.rs b/src/agent/provider/anthropic/utils.rs index 84f3726..e124d17 100644 --- a/src/agent/provider/anthropic/utils.rs +++ b/src/agent/provider/anthropic/utils.rs @@ -28,6 +28,13 @@ pub(super) fn create_anthropic_message_request(llm_messages: Vec) -> }, }] } + LLMMessageContent::File(file_details) => { + tracing::warn!( + "The Anthropic provider's library does not support file/document content. This file message ({}) will be skipped.", + file_details.filename(), + ); + continue; + } }; let message = Message { role, content }; diff --git a/src/agent/provider/openai/utils.rs b/src/agent/provider/openai/utils.rs index 03925e3..e0fe60e 100644 --- a/src/agent/provider/openai/utils.rs +++ b/src/agent/provider/openai/utils.rs @@ -1,6 +1,6 @@ use async_openai::types::responses::{ - EasyInputContent, EasyInputMessage, ImageDetail, InputContent, InputImageContent, InputItem, - InputParam, MessageType, Role, + EasyInputContent, EasyInputMessage, ImageDetail, InputContent, InputFileArgs, + InputImageContent, InputItem, InputParam, MessageType, Role, }; use crate::conversation::llm::{ @@ -35,6 +35,21 @@ pub fn convert_llm_messages_to_openai_response_input( file_id: None, })]) } + LLMMessageContent::File(file_details) => { + let file_data = format!( + "data:{};base64,{}", + file_details.mime, + base64_encode(&file_details.data) + ); + + let file_content = InputFileArgs::default() + .file_data(file_data) + .filename(file_details.filename()) + .build() + .expect("Failed to build InputFileContent"); + + EasyInputContent::ContentList(vec![InputContent::InputFile(file_content)]) + } }; items.push(InputItem::EasyMessage(EasyInputMessage { diff --git a/src/agent/provider/openai_compat/utils.rs b/src/agent/provider/openai_compat/utils.rs index c58a89e..b560bd8 100644 --- a/src/agent/provider/openai_compat/utils.rs +++ b/src/agent/provider/openai_compat/utils.rs @@ -40,6 +40,12 @@ fn convert_llm_message_to_openai_message(llm_message: LLMMessage) -> Option { + tracing::warn!( + "The OpenAI-compat provider's library does not support file content. This file message will be skipped." + ); + None + } } } diff --git a/src/controller/chat_completion/mod.rs b/src/controller/chat_completion/mod.rs index 3787b39..bed9622 100644 --- a/src/controller/chat_completion/mod.rs +++ b/src/controller/chat_completion/mod.rs @@ -41,6 +41,8 @@ pub enum ChatCompletionControllerType { Image, + File, + ThreadMention, ReplyMention, } @@ -419,7 +421,8 @@ async fn handle_stage_text_generation( | ChatCompletionControllerType::TextMention | ChatCompletionControllerType::TextDirect | ChatCompletionControllerType::Audio - | ChatCompletionControllerType::Image => { + | ChatCompletionControllerType::Image + | ChatCompletionControllerType::File => { Some(message_context.combined_admin_and_user_regexes()) } diff --git a/src/controller/determination/mod.rs b/src/controller/determination/mod.rs index ba43d4c..795aefa 100644 --- a/src/controller/determination/mod.rs +++ b/src/controller/determination/mod.rs @@ -58,6 +58,18 @@ pub fn determine_controller( ) } } + MessagePayload::File(_file_message_content) => { + let prefix_requirement_type = message_context + .room_config_context() + .text_generation_prefix_requirement_type(); + + match prefix_requirement_type { + TextGenerationPrefixRequirementType::CommandPrefix => ControllerType::Ignore, + TextGenerationPrefixRequirementType::No => { + ControllerType::ChatCompletion(ChatCompletionControllerType::File) + } + } + } MessagePayload::Audio(_) => { ControllerType::ChatCompletion(ChatCompletionControllerType::Audio) } diff --git a/src/conversation/llm/entity.rs b/src/conversation/llm/entity.rs index b76a204..27f41fb 100644 --- a/src/conversation/llm/entity.rs +++ b/src/conversation/llm/entity.rs @@ -1,5 +1,7 @@ use chrono::{DateTime, Utc}; -use mxlink::matrix_sdk::ruma::events::room::message::ImageMessageEventContent; +use mxlink::matrix_sdk::ruma::events::room::message::{ + FileMessageEventContent, ImageMessageEventContent, +}; use mxlink::mime::Mime; use crate::agent::provider::ImageSource; @@ -48,10 +50,35 @@ impl From for ImageSource { } } +#[derive(Debug, Clone)] +pub struct FileDetails { + pub event_content: FileMessageEventContent, + pub mime: Mime, + pub data: Vec, +} + +impl FileDetails { + pub fn new(event_content: FileMessageEventContent, mime: Mime, data: Vec) -> Self { + Self { + event_content, + mime, + data, + } + } + + pub fn filename(&self) -> String { + self.event_content + .filename + .clone() + .unwrap_or(self.event_content.body.clone()) + } +} + #[derive(Debug, Clone)] pub enum MessageContent { Text(String), Image(ImageDetails), + File(FileDetails), } impl PartialEq for MessageContent { @@ -62,6 +89,7 @@ impl PartialEq for MessageContent { // We can probably do better than this by inspecting `.event_conten1t.source`, but for now this is good enough. a.filename() == b.filename() } + (MessageContent::File(a), MessageContent::File(b)) => a.filename() == b.filename(), _ => false, } } diff --git a/src/conversation/llm/tokenization.rs b/src/conversation/llm/tokenization.rs index edb69cd..25069f6 100644 --- a/src/conversation/llm/tokenization.rs +++ b/src/conversation/llm/tokenization.rs @@ -74,6 +74,7 @@ fn calculate_token_size_for_message(bpe: &CoreBPE, model: &str, message: &Messag let text_length = match &message.content { MessageContent::Text(text) => bpe.encode_with_special_tokens(text).len() as i32, MessageContent::Image(..) => 0, + MessageContent::File(..) => 0, }; (text_length + role_length + tokens_per_message + tokens_per_name) as u32 diff --git a/src/conversation/llm/utils.rs b/src/conversation/llm/utils.rs index c5b109f..b9cfc06 100644 --- a/src/conversation/llm/utils.rs +++ b/src/conversation/llm/utils.rs @@ -1,6 +1,6 @@ use mxlink::matrix_sdk::ruma::OwnedUserId; -use super::entity::{Author, ImageDetails, Message, MessageContent}; +use super::entity::{Author, FileDetails, ImageDetails, Message, MessageContent}; use crate::conversation::matrix::{MatrixMessage, MatrixMessageContent}; use crate::utils::text_to_speech as text_to_speech_utils; @@ -32,6 +32,15 @@ fn convert_bot_message(matrix_message: &MatrixMessage) -> Option { )), timestamp: matrix_message.timestamp.to_owned(), }), + MatrixMessageContent::File(file_content, mime_type, media_bytes) => Some(Message { + author: Author::Assistant, + content: MessageContent::File(FileDetails::new( + file_content.clone(), + mime_type.clone(), + media_bytes.clone(), + )), + timestamp: matrix_message.timestamp.to_owned(), + }), } } @@ -90,5 +99,14 @@ fn convert_user_message(matrix_message: &MatrixMessage) -> Option { )), timestamp: matrix_message.timestamp.to_owned(), }), + MatrixMessageContent::File(file_content, mime_type, media_bytes) => Some(Message { + author: Author::User, + content: MessageContent::File(FileDetails::new( + file_content.clone(), + mime_type.clone(), + media_bytes.clone(), + )), + timestamp: matrix_message.timestamp.to_owned(), + }), } } diff --git a/src/conversation/matrix/entity.rs b/src/conversation/matrix/entity.rs index 28c3f14..18bd85b 100644 --- a/src/conversation/matrix/entity.rs +++ b/src/conversation/matrix/entity.rs @@ -2,7 +2,9 @@ use chrono::{DateTime, Utc}; use regex::Regex; use mxlink::matrix_sdk::ruma::OwnedUserId; -use mxlink::matrix_sdk::ruma::events::room::message::ImageMessageEventContent; +use mxlink::matrix_sdk::ruma::events::room::message::{ + FileMessageEventContent, ImageMessageEventContent, +}; use mxlink::mime::Mime; #[derive(Clone)] @@ -18,6 +20,7 @@ pub enum MatrixMessageContent { Text(String), Notice(String), Image(ImageMessageEventContent, Mime, Vec), + File(FileMessageEventContent, Mime, Vec), } #[derive(Clone)] diff --git a/src/conversation/matrix/utils/mod.rs b/src/conversation/matrix/utils/mod.rs index ac5b3f5..f5f3e77 100644 --- a/src/conversation/matrix/utils/mod.rs +++ b/src/conversation/matrix/utils/mod.rs @@ -234,6 +234,7 @@ pub async fn convert_matrix_native_event_to_matrix_message( MessageType::Text(text_content) => (text_content.body.clone(), false), MessageType::Notice(notice_content) => (notice_content.body.clone(), true), MessageType::Image(image_content) => (image_content.body.clone(), false), + MessageType::File(file_content) => (file_content.body.clone(), false), _ => return Ok(None), }; @@ -291,6 +292,49 @@ pub async fn convert_matrix_native_event_to_matrix_message( })); } + if let MessageType::File(file_content) = &room_message.msgtype { + let media_request = mxlink::matrix_sdk::media::MediaRequestParameters { + source: file_content.source.to_owned(), + format: mxlink::matrix_sdk::media::MediaFormat::File, + }; + + let file_name = file_content + .filename + .clone() + .unwrap_or(file_content.body.clone()); + + let mime_type = file_content + .info + .as_ref() + .and_then(|info| info.mimetype.clone()) + .and_then(|mimetype| mimetype.parse::().ok()) + .unwrap_or_else(|| get_mime_type_from_file_name(&file_name)); + + tracing::debug!("Determined mime type {} for file {}", mime_type, file_name); + + let span = tracing::debug_span!("get_media_content", file_name = %file_name, mime_type = %mime_type); + + let media_bytes = matrix_link + .client() + .media() + .get_media_content(&media_request, true) + .instrument(span) + .await?; + + tracing::debug!( + "Downloaded {} bytes for file {}", + media_bytes.len(), + file_name + ); + + return Ok(Some(MatrixMessage { + sender_id: matrix_native_event.sender().to_owned(), + content: MatrixMessageContent::File(file_content.clone(), mime_type, media_bytes), + mentioned_users, + timestamp, + })); + } + Ok(Some(MatrixMessage { sender_id: matrix_native_event.sender().to_owned(), content: if is_notice { diff --git a/src/entity/message_payload.rs b/src/entity/message_payload.rs index 12a4463..1d15421 100644 --- a/src/entity/message_payload.rs +++ b/src/entity/message_payload.rs @@ -1,5 +1,6 @@ use mxlink::matrix_sdk::ruma::events::room::message::{ - AudioMessageEventContent, ImageMessageEventContent, MessageType, TextMessageEventContent, + AudioMessageEventContent, FileMessageEventContent, ImageMessageEventContent, MessageType, + TextMessageEventContent, }; use mxlink::matrix_sdk::ruma::{OwnedEventId, OwnedUserId}; @@ -30,6 +31,7 @@ pub enum MessagePayload { Text(TextMessageEventContent), Audio(AudioMessageEventContent), Image(ImageMessageEventContent), + File(FileMessageEventContent), Reaction { key: String, @@ -57,6 +59,7 @@ impl TryInto for MessageType { MessagePayload::Audio(audio_content) } MessageType::Image(image_content) => MessagePayload::Image(image_content), + MessageType::File(file_content) => MessagePayload::File(file_content), other => { return Err(format!("Unsupported message type: {:?}", other)); } diff --git a/src/utils/mime.rs b/src/utils/mime.rs index 96b2998..bd4988a 100644 --- a/src/utils/mime.rs +++ b/src/utils/mime.rs @@ -29,6 +29,18 @@ pub fn get_mime_type_from_file_name(file_name: &str) -> mime::Mime { "bmp" => "image/bmp".parse().unwrap(), "heic" | "heif" => "image/heic".parse().unwrap(), "avif" => "image/avif".parse().unwrap(), + "pdf" => "application/pdf".parse().unwrap(), + "txt" | "log" => mime::TEXT_PLAIN, + "csv" => mime::TEXT_CSV, + "json" => mime::APPLICATION_JSON, + "xml" => "application/xml".parse().unwrap(), + "html" | "htm" => mime::TEXT_HTML, + "md" | "markdown" => "text/markdown".parse().unwrap(), + "yaml" | "yml" => "application/yaml".parse().unwrap(), + "toml" => "application/toml".parse().unwrap(), + "rs" | "py" | "js" | "ts" | "go" | "c" | "cpp" | "h" | "java" | "rb" | "sh" | "css" => { + mime::TEXT_PLAIN + } _ => mime::APPLICATION_OCTET_STREAM, } }