From 2f9c3dfce012765001e0c2eda0cf726fb6c37b35 Mon Sep 17 00:00:00 2001
From: Slavi Pantaleev <slavi@devture.com>
Date: Sat, 10 May 2025 10:24:55 +0300
Subject: [PATCH] Use patched `anthropic-rs` library to add Vision support to
 text conversations

Related to: https://github.com/AbdelStark/anthropic-rs/pull/11
---
 Cargo.lock                                 |  3 +--
 Cargo.toml                                 |  2 +-
 src/agent/provider/anthropic/controller.rs | 11 ++------
 src/agent/provider/anthropic/utils.rs      | 29 ++++++++--------------
 4 files changed, 15 insertions(+), 30 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index e22f4a2..1b7a787 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -90,8 +90,7 @@ dependencies = [
 [[package]]
 name = "anthropic"
 version = "0.0.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "092086afb8afa8a2bf24820213e22a57662eace03bd6b3c2680f6baf022ef6ba"
+source = "git+https://github.com/etkecc/anthropic-rs.git?branch=fix-content-block-image#58d8492450304ca9c4d0fdbfd023668231862d2c"
 dependencies = [
  "backoff",
  "config",
diff --git a/Cargo.toml b/Cargo.toml
index 4e75e62..ba60eb8 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -15,7 +15,7 @@ name = "baibot"
 path = "src/lib.rs"
 
 [dependencies]
-anthropic = "=0.0.8"
+anthropic = { git = "https://github.com/etkecc/anthropic-rs.git", branch = "fix-content-block-image" }
 anyhow = "1.0.*"
 async-openai = { git = "https://github.com/etkecc/async-openai", branch = "async-openai-v0.28.1-patched" }
 base64 = "0.22.*"
diff --git a/src/agent/provider/anthropic/controller.rs b/src/agent/provider/anthropic/controller.rs
index 98f3d47..991ad8e 100644
--- a/src/agent/provider/anthropic/controller.rs
+++ b/src/agent/provider/anthropic/controller.rs
@@ -176,15 +176,8 @@ impl ControllerTrait for Controller {
                 ContentBlock::Text { text } => {
                     text_parts.push(text);
                 }
-                ContentBlock::Image {
-                    source,
-                    media_type,
-                    data: _,
-                } => {
-                    text_parts.push(format!(
-                        "The model responded with an image of type {}: {}",
-                        media_type, source
-                    ));
+                ContentBlock::Image { .. } => {
+                    text_parts.push("The model responded with an image".to_string());
                 }
             }
         }
diff --git a/src/agent/provider/anthropic/utils.rs b/src/agent/provider/anthropic/utils.rs
index c246d95..040c2bf 100644
--- a/src/agent/provider/anthropic/utils.rs
+++ b/src/agent/provider/anthropic/utils.rs
@@ -1,4 +1,4 @@
-use anthropic::types::{ContentBlock, Message, MessagesRequest, MessagesRequestBuilder, Role};
+use anthropic::types::{ContentBlock, ImageSource, Message, MessagesRequest, MessagesRequestBuilder, Role};
 
 use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent};
 
@@ -15,27 +15,20 @@ pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) ->
         };
 
         let content = match &message.content {
-            LLMMessageContent::Text(text) => Some(vec![ContentBlock::Text { text: text.clone() }]),
-            LLMMessageContent::Image(_image_details) => {
-                // This cannot be implemented yet, because the Anthropic library does not support it.
-                // The code below requires the library to be forked and changed a bit.
-                // vec![ContentBlock::Image {
-                //     source: ImageSource {
-                //         r#type: "base64".to_string(),
-                //         media_type: mime_type.to_string(),
-                //         data: crate::utils::base64::base64_encode(image_details.data),
-                //     },
-                // }]
-                tracing::warn!("Image content is not supported by the Anthropic library yet. Skipping it.");
-                None
+            LLMMessageContent::Text(text) => vec![ContentBlock::Text { text: text.clone() }],
+            LLMMessageContent::Image(image_details) => {
+                vec![ContentBlock::Image {
+                    source: ImageSource::Base64 {
+                        media_type: image_details.mime.to_string(),
+                        data: crate::utils::base64::base64_encode(&image_details.data),
+                    },
+                }]
             }
         };
 
-        if let Some(content) = content {
-            let message = Message { role, content };
+        let message = Message { role, content };
 
-            messages.push(message);
-        }
+        messages.push(message);
     }
 
     MessagesRequestBuilder::default()