Initial work on Vision support in text conversations and Image Editing
This is a huge patch which does some major refactoring like: - renaming "Image Generation" to "Image Creation" in most places, to better match its new command (`!bai image create`) - relocating image creation command (`!bai image` -> `!bai image create`), so it wouldn't conflict with the new image editing command (`!bai image edit`) - introducing a new image editing command (`!bai image edit`), which is meant to work only with the OpenAI provider, but doesn't fully work yet due to https://github.com/64bit/async-openai/issues/364, though a next patch will fix it - adding support for reading images off of Matrix conversations and forwarding them to text conversations. Works for OpenAI, but not for Anthropic yet (requires custom patches) and not for OpenAI-Compat (no support for images there) - relocating some utils around (base64, mime)
This commit is contained in:
@@ -1,17 +0,0 @@
|
||||
use mxlink::mime;
|
||||
|
||||
pub fn get_file_extension(mime_type: &mime::Mime) -> String {
|
||||
match (mime_type.type_(), mime_type.subtype()) {
|
||||
(mime::AUDIO, mime::BASIC) => "au",
|
||||
(mime::AUDIO, mime::MPEG) => "mp3",
|
||||
(mime::AUDIO, mime::MP4) => "m4a",
|
||||
(mime::AUDIO, mime::OGG) => "ogg",
|
||||
(mime::IMAGE, mime::BMP) => "bmp",
|
||||
(mime::IMAGE, mime::GIF) => "gif",
|
||||
(mime::IMAGE, mime::JPEG) => "jpg",
|
||||
(mime::IMAGE, mime::PNG) => "png",
|
||||
(mime::IMAGE, mime::SVG) => "svg",
|
||||
_ => "bin",
|
||||
}
|
||||
.to_string()
|
||||
}
|
||||
@@ -6,7 +6,6 @@ use crate::{
|
||||
};
|
||||
|
||||
pub mod agent;
|
||||
pub(super) mod mime;
|
||||
pub mod text_to_speech;
|
||||
|
||||
pub async fn get_text_body_or_complain<'a>(
|
||||
|
||||
@@ -3,7 +3,7 @@ use mxlink::{MatrixLink, MessageResponseType};
|
||||
|
||||
use tracing::Instrument;
|
||||
|
||||
use crate::controller::utils::mime::get_file_extension;
|
||||
use crate::utils::mime::get_file_extension;
|
||||
use crate::{
|
||||
Bot,
|
||||
agent::{AgentInstance, AgentPurpose, ControllerTrait, provider::TextToSpeechParams},
|
||||
|
||||
Reference in New Issue
Block a user