Initial work on Vision support in text conversations and Image Editing
This is a huge patch which does some major refactoring like: - renaming "Image Generation" to "Image Creation" in most places, to better match its new command (`!bai image create`) - relocating image creation command (`!bai image` -> `!bai image create`), so it wouldn't conflict with the new image editing command (`!bai image edit`) - introducing a new image editing command (`!bai image edit`), which is meant to work only with the OpenAI provider, but doesn't fully work yet due to https://github.com/64bit/async-openai/issues/364, though a next patch will fix it - adding support for reading images off of Matrix conversations and forwarding them to text conversations. Works for OpenAI, but not for Anthropic yet (requires custom patches) and not for OpenAI-Compat (no support for images there) - relocating some utils around (base64, mime)
This commit is contained in:
65
src/agent/provider/entity/image.rs
Normal file
65
src/agent/provider/entity/image.rs
Normal file
@@ -0,0 +1,65 @@
|
||||
use mxlink::mime;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct ImageGenerationParams {
|
||||
pub size_override: Option<String>,
|
||||
|
||||
pub cheaper_model_switching_allowed: bool,
|
||||
|
||||
pub cheaper_quality_switching_allowed: bool,
|
||||
}
|
||||
|
||||
impl ImageGenerationParams {
|
||||
pub fn with_size_override(mut self, value: Option<String>) -> Self {
|
||||
self.size_override = value;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_cheaper_model_switching_allowed(mut self, value: bool) -> Self {
|
||||
self.cheaper_model_switching_allowed = value;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_cheaper_quality_switching_allowed(mut self, value: bool) -> Self {
|
||||
self.cheaper_quality_switching_allowed = value;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ImageGenerationResult {
|
||||
pub bytes: Vec<u8>,
|
||||
pub mime_type: mime::Mime,
|
||||
pub revised_prompt: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct ImageEditParams {
|
||||
}
|
||||
|
||||
pub struct ImageEditResult {
|
||||
pub bytes: Vec<u8>,
|
||||
pub mime_type: mime::Mime,
|
||||
}
|
||||
|
||||
pub struct ImageSource {
|
||||
pub filename: String,
|
||||
pub bytes: Vec<u8>,
|
||||
pub mime_type: mime::Mime,
|
||||
}
|
||||
|
||||
impl ImageSource {
|
||||
pub fn new(filename: String, bytes: Vec<u8>, mime_type: mime::Mime) -> Self {
|
||||
Self { filename, bytes, mime_type }
|
||||
}
|
||||
}
|
||||
|
||||
impl Into<async_openai::types::ImageInput> for ImageSource {
|
||||
fn into(self) -> async_openai::types::ImageInput {
|
||||
async_openai::types::ImageInput{
|
||||
source: async_openai::types::InputSource::VecU8 {
|
||||
filename: self.filename,
|
||||
vec: self.bytes,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
#[derive(Default)]
|
||||
pub struct ImageGenerationParams {
|
||||
pub size_override: Option<String>,
|
||||
|
||||
pub cheaper_model_switching_allowed: bool,
|
||||
|
||||
pub cheaper_quality_switching_allowed: bool,
|
||||
}
|
||||
|
||||
impl ImageGenerationParams {
|
||||
pub fn with_size_override(mut self, value: Option<String>) -> Self {
|
||||
self.size_override = value;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_cheaper_model_switching_allowed(mut self, value: bool) -> Self {
|
||||
self.cheaper_model_switching_allowed = value;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_cheaper_quality_switching_allowed(mut self, value: bool) -> Self {
|
||||
self.cheaper_quality_switching_allowed = value;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
pub struct ImageGenerationResult {
|
||||
pub bytes: Vec<u8>,
|
||||
pub mime_type: mxlink::mime::Mime,
|
||||
pub revised_prompt: Option<String>,
|
||||
}
|
||||
@@ -1,12 +1,12 @@
|
||||
mod agent_provider;
|
||||
mod image_generation;
|
||||
mod image;
|
||||
mod ping;
|
||||
mod speech_to_text;
|
||||
mod text_generation;
|
||||
mod text_to_speech;
|
||||
|
||||
pub use agent_provider::{AgentProvider, AgentProviderInfo};
|
||||
pub use image_generation::{ImageGenerationParams, ImageGenerationResult};
|
||||
pub use image::{ImageGenerationParams, ImageGenerationResult, ImageEditParams, ImageEditResult, ImageSource};
|
||||
pub use ping::PingResult;
|
||||
pub use speech_to_text::{SpeechToTextParams, SpeechToTextResult};
|
||||
pub use text_generation::{
|
||||
|
||||
Reference in New Issue
Block a user