Initial work on Vision support in text conversations and Image Editing

This is a huge patch which does some major refactoring like:

- renaming "Image Generation" to "Image Creation" in most places,
  to better match its new command (`!bai image create`)

- relocating image creation command (`!bai image` -> `!bai image create`),
  so it wouldn't conflict with the new image editing command (`!bai image edit`)

- introducing a new image editing command (`!bai image edit`), which
  is meant to work only with the OpenAI provider, but doesn't fully work yet
  due to https://github.com/64bit/async-openai/issues/364, though a next patch will fix it

- adding support for reading images off of Matrix conversations and forwarding them to
  text conversations. Works for OpenAI, but not for Anthropic yet
  (requires custom patches) and not for OpenAI-Compat (no support for
  images there)

- relocating some utils around (base64, mime)
This commit is contained in:
Slavi Pantaleev
2025-05-10 09:18:01 +03:00
parent e0dcc39a72
commit 8f86289373
57 changed files with 1074 additions and 335 deletions

9
src/utils/base64.rs Normal file
View File

@@ -0,0 +1,9 @@
use base64::{Engine as _, engine::general_purpose::STANDARD};
pub(crate) fn base64_decode(base64_string: &str) -> Result<Vec<u8>, base64::DecodeError> {
STANDARD.decode(base64_string)
}
pub(crate) fn base64_encode(data: &[u8]) -> String {
STANDARD.encode(data)
}

34
src/utils/mime.rs Normal file
View File

@@ -0,0 +1,34 @@
use mxlink::mime;
pub fn get_file_extension(mime_type: &mime::Mime) -> String {
match (mime_type.type_(), mime_type.subtype()) {
(mime::AUDIO, mime::BASIC) => "au",
(mime::AUDIO, mime::MPEG) => "mp3",
(mime::AUDIO, mime::MP4) => "m4a",
(mime::AUDIO, mime::OGG) => "ogg",
(mime::IMAGE, mime::BMP) => "bmp",
(mime::IMAGE, mime::GIF) => "gif",
(mime::IMAGE, mime::JPEG) => "jpg",
(mime::IMAGE, mime::PNG) => "png",
(mime::IMAGE, mime::SVG) => "svg",
_ => "bin",
}
.to_string()
}
pub fn get_mime_type_from_file_name(file_name: &str) -> mime::Mime {
let extension = file_name.rsplit('.').next().unwrap_or("");
match extension.to_lowercase().as_str() {
"jpg" | "jpeg" => mime::IMAGE_JPEG,
"png" => mime::IMAGE_PNG,
"gif" => mime::IMAGE_GIF,
"webp" => "image/webp".parse().unwrap(),
"svg" => mime::IMAGE_SVG,
"tiff" | "tif" => "image/tiff".parse().unwrap(),
"bmp" => "image/bmp".parse().unwrap(),
"heic" | "heif" => "image/heic".parse().unwrap(),
"avif" => "image/avif".parse().unwrap(),
_ => mime::APPLICATION_OCTET_STREAM,
}
}

View File

@@ -1,3 +1,5 @@
pub mod status;
pub mod text;
pub mod text_to_speech;
pub(crate) mod mime;
pub(crate) mod base64;