Release 1.4.0

It seems like Element iOS benefits from this.

CHANGELOG.md

@@ -1,8 +1,24 @@
+# (2024-11-19) Version 1.4.0
+
+- (**Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.4.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.8.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.8.0)). Once you run this version at least once and your matrix-sdk datastore gets upgraded to the new schema, **you will not be able to downgrade to older baibot versions** (based on the older matrix-sdk), unless you start with an empty datastore.
+
+- (**Bugfix**) Add missing typing notices sending functionality while generating images ([9d166e35ba](https://github.com/etkecc/baibot/commit/9d166e35ba6fc0daaf69318870e92436f3302056))
+
+
+# (2024-11-12) Version 1.3.2
+
+Dependency updates.
+
+
+# (2024-10-03) Version 1.3.1
+
+- (**Improvement**) Improves fallback user mentions support for old clients (like Element iOS) which use the bot's display name (not its full Matrix User ID). ([d9a045a5e4](https://github.com/etkecc/baibot/commit/d9a045a5e41d2b99694f92ec9e90f47529546d89))
+
+
 # (2024-10-03) Version 1.3.0
 
 **TLDR**: you can now use OpenAI's [o1](https://platform.openai.com/docs/models/o1) models, benefit from [prompt caching](https://platform.openai.com/docs/guides/prompt-caching) and mention the bot again from old clients lacking proper [user mentions support](https://spec.matrix.org/latest/client-server-api/#user-and-room-mentions) (like Element iOS).
 
-
 - (**Feature**) Introduces a new `baibot_conversation_start_time_utc` [prompt variable](./docs/configuration/text-generation.md#️-prompt-override) which is not a moving target (like the `baibot_now_utc` variable) and allows [prompt caching](https://platform.openai.com/docs/guides/prompt-caching) to work. All default/sample configs have been adjusted to make use of this new variable, but users need to adjust your existing dynamically-created agents to start using it. ([85e66406dc](https://github.com/etkecc/baibot/commit/85e66406dc6f430741c7819f420e2df4ae6e8d3b))
 
 - (**Improvement**) Allows for the `max_response_tokens` configuration value for the [OpenAI provider](./docs/providers.md#openai) to be set to `null` to allow [o1](https://platform.openai.com/docs/models/o1) models (which do not support `max_response_tokens`) to be used. See the new o1 sample config [here](./docs/sample-provider-configs/openai-o1.yml). ([db9422740c](https://github.com/etkecc/baibot/commit/db9422740ceca32956d9628b6326b8be206344e2))

Cargo.toml

@@ -7,7 +7,7 @@ license = "AGPL-3.0-or-later"
 readme = "README.md"
 keywords = ["matrix", "chat", "bot", "AI", "LLM"]
 include = ["/etc/assets/baibot-torso-768.png", "/src", "/README.md", "/CHANGELOG.md", "/LICENSE"]
-version = "1.3.0"
+version = "1.4.0"
 edition = "2021"
 
 [lib]
@@ -17,22 +17,22 @@ path = "src/lib.rs"
 [dependencies]
 anthropic-rs = "0.1.*"
 anyhow = "1.0.*"
-async-openai = "0.24.*"
+async-openai = "0.26.*"
 base64 = "0.22.*"
 chrono = { version = "0.4.*", default-features = false, features = ["std", "now"] }
 # We'd rather not depend on this, but we cannot use the ruma-events EventContent macro without it.
-matrix-sdk = { version = "0.7.1", default-features = false }
+matrix-sdk = { version = "0.8.0", default-features = false }
 mxidwc = "1.0.*"
-mxlink = ">=1.3.0"
+mxlink = ">=1.4.0"
 etke_openai_api_rust = "0.1.*"
 quick_cache = "0.6.*"
 regex = "1.11.*"
 serde = { version = "1.0.*", features = ["derive"], default-features = false }
 serde_json = "1.0.*"
 serde_yaml = "0.9.*"
-tempfile = "3.13.*"
+tempfile = "3.14.*"
-tiktoken-rs = { version = "0.5.*", features = ["async-openai"] }
+tiktoken-rs = { version = "0.6.*", features = ["async-openai"] }
-tokio = { version = "1.40.*", features = ["rt", "rt-multi-thread", "macros"] }
+tokio = { version = "1.41.*", features = ["rt", "rt-multi-thread", "macros"] }
 tracing = "0.1.*"
 tracing-subscriber = { version = "0.3.*", features = ["env-filter"] }
 url = "2.5.*"

Dockerfile

@@ -4,7 +4,7 @@
 #                                     #
 #######################################
 
-FROM docker.io/rust:1.81.0-slim-bookworm AS build
+FROM docker.io/rust:1.82.0-slim-bookworm AS build
 
 RUN apt-get update && apt-get install -y build-essential pkg-config libssl-dev libsqlite3-dev
 

Dockerfile.ci

@@ -4,7 +4,7 @@
 #                                     #
 #######################################
 
-FROM docker.io/rust:1.81.0-slim-bookworm AS build
+FROM docker.io/rust:1.82.0-slim-bookworm AS build
 
 RUN apt-get update && apt-get install -y build-essential pkg-config libssl-dev libsqlite3-dev
 

etc/services/core/compose.yml

@@ -13,7 +13,7 @@ services:
     - /etc/passwd:/etc/passwd:ro
 
   synapse:
-    image: ghcr.io/element-hq/synapse:v1.116.0
+    image: ghcr.io/element-hq/synapse:v1.118.0
     user: "${UID}:${GID}"
     restart: unless-stopped
     entrypoint: python
@@ -26,7 +26,7 @@ services:
     - ./synapse/media-store:/media-store
 
   element-web:
-    image: docker.io/vectorim/element-web:v1.11.79
+    image: docker.io/vectorim/element-web:v1.11.84
     user: "${UID}:${GID}"
     restart: unless-stopped
     ports:

etc/services/ollama/compose.yml

@@ -1,6 +1,6 @@
 services:
   ollama:
-    image: docker.io/ollama/ollama:0.3.11
+    image: docker.io/ollama/ollama:0.4.1
     restart: unless-stopped
     ports:
       - "${SERVICE_OLLAMA_BIND_PORT_HTTP}:11434"

src/bot/implementation.rs

@@ -1,7 +1,7 @@
 use std::sync::Arc;
 use std::{future::Future, pin::Pin};
 
-use mxlink::matrix_sdk::media::{MediaFormat, MediaRequest};
+use mxlink::matrix_sdk::media::{MediaFormat, MediaRequestParameters};
 use mxlink::matrix_sdk::ruma::{
     events::room::MediaSource, MilliSecondsSinceUnixEpoch, OwnedUserId,
 };
@@ -172,6 +172,24 @@ impl Bot {
         self.matrix_link().user_id()
     }
 
+    pub(crate) async fn user_display_name_in_room(&self, room: &Room) -> Option<String> {
+        let bot_display_name = self
+            .room_display_name_fetcher()
+            .own_display_name_in_room(room)
+            .await;
+
+        match bot_display_name {
+            Ok(value) => value,
+            Err(err) => {
+                tracing::warn!(
+                    ?err,
+                    "Failed to fetch bot display name. Proceeding without it"
+                );
+                None
+            }
+        }
+    }
+
     pub(crate) fn reacting(&self) -> super::reacting::Reacting {
         super::reacting::Reacting::new(self.clone())
     }
@@ -269,7 +287,7 @@ impl Bot {
         let desired_display_name = self.inner.config.user.name.clone();
 
         let profile = account
-            .get_profile()
+            .fetch_user_profile()
             .await
             .map_err(|e| anyhow::anyhow!("Failed fetching profile: {:?}", e))?;
 
@@ -292,7 +310,7 @@ impl Bot {
 
         let should_update_avatar = match &profile.avatar_url {
             Some(avatar_url) => {
-                let request = MediaRequest {
+                let request = MediaRequestParameters {
                     source: MediaSource::Plain(avatar_url.to_owned()),
                     format: MediaFormat::File,
                 };

src/bot/messaging.rs

@@ -239,8 +239,11 @@ impl Messaging {
             }
         };
 
+        let bot_display_name = self.bot.user_display_name_in_room(&room).await;
+
         let interaction_context = determine_interaction_context_for_room_event(
             self.bot.user_id(),
+            &bot_display_name,
             &room,
             &event,
             &payload,
@@ -279,7 +282,8 @@ impl Messaging {
             self.bot.admin_pattern_regexes().clone(),
             trigger_event_info,
             interaction_context.thread_info.clone(),
-        );
+        )
+        .with_bot_display_name(bot_display_name);
 
         let controller_type = crate::controller::determine_controller(
             self.bot.command_prefix(),

src/bot/reacting.rs

@@ -1,8 +1,8 @@
 use mxlink::matrix_sdk::{
     ruma::{
         events::{
-            room::message::Relation, AnyMessageLikeEvent, AnySyncTimelineEvent, AnyTimelineEvent,
+            room::message::Relation, AnySyncMessageLikeEvent, AnySyncTimelineEvent,
-            MessageLikeEvent,
+            SyncMessageLikeEvent,
         },
         OwnedEventId, OwnedUserId,
     },
@@ -139,7 +139,7 @@ impl Reacting {
             }
         };
 
-        let reacted_to_event_any_timeline_event = match reacted_to_event.event.deserialize() {
+        let reacted_to_event_any_timeline_event = match reacted_to_event.raw().deserialize() {
             Ok(value) => value,
             Err(err) => {
                 tracing::error!(
@@ -154,7 +154,7 @@ impl Reacting {
         let reacted_to_event_sender_id: OwnedUserId =
             reacted_to_event_any_timeline_event.sender().to_owned();
 
-        let AnyTimelineEvent::MessageLike(reacted_to_event_message_like) =
+        let AnySyncTimelineEvent::MessageLike(reacted_to_event_message_like) =
             reacted_to_event_any_timeline_event
         else {
             tracing::debug!(
@@ -164,7 +164,7 @@ impl Reacting {
             return Ok(());
         };
 
-        let AnyMessageLikeEvent::RoomMessage(reacted_to_event_room_message) =
+        let AnySyncMessageLikeEvent::RoomMessage(reacted_to_event_room_message) =
             reacted_to_event_message_like
         else {
             tracing::debug!(
@@ -174,7 +174,7 @@ impl Reacting {
             return Ok(());
         };
 
-        let MessageLikeEvent::Original(reacted_to_event_room_message_original) =
+        let SyncMessageLikeEvent::Original(reacted_to_event_room_message_original) =
             reacted_to_event_room_message
         else {
             tracing::debug!(?reacted_to_event_id, "Ignoring redacted reacted-to event",);

src/controller/chat_completion/mod.rs

@@ -383,24 +383,10 @@ async fn handle_stage_text_generation(
         _ => vec![],
     };
 
-    let bot_display_name = bot
+    let bot_user_prefixes_to_strip = create_list_of_bot_user_prefixes_to_strip(
-        .room_display_name_fetcher()
+        bot.user_id(),
-        .own_display_name_in_room(message_context.room())
+        message_context.bot_display_name(),
-        .await;
+    );
-
-    let bot_display_name = match bot_display_name {
-        Ok(value) => value,
-        Err(err) => {
-            tracing::warn!(
-                ?err,
-                "Failed to fetch bot display name. Proceeding without it"
-            );
-            None
-        }
-    };
-
-    let bot_user_prefixes_to_strip =
-        create_list_of_bot_user_prefixes_to_strip(bot.user_id(), &bot_display_name);
 
     let allowed_users = match controller_type {
         // Regular chat completion only operates on messages from allowed users.
@@ -589,7 +575,7 @@ async fn handle_stage_speech_to_text_actual_transcribing(
 ) -> anyhow::Result<OwnedEventId> {
     let src = &audio_content.source;
 
-    let media_request = mxlink::matrix_sdk::media::MediaRequest {
+    let media_request = mxlink::matrix_sdk::media::MediaRequestParameters {
         source: src.to_owned(),
         format: mxlink::matrix_sdk::media::MediaFormat::File,
     };

src/controller/image/generation.rs

@@ -34,6 +34,8 @@ pub async fn handle_image(
         return Ok(());
     };
 
+    let _typing_notice_guard = bot.start_typing_notice(message_context.room()).await;
+
     let params = MatrixMessageProcessingParams::new(
         bot.user_id().to_owned(),
         Some(message_context.combined_admin_and_user_regexes()),

src/conversation/matrix/room_event_fetcher.rs

@@ -22,7 +22,7 @@ impl RoomEventFetcher {
         room: &Room,
     ) -> mxlink::matrix_sdk::Result<TimelineEvent> {
         let Some(lru_cache) = &self.lru_cache else {
-            return room.event(event_id).await;
+            return room.event(event_id, None).await;
         };
 
         let guard = lru_cache.get_value_or_guard_async(event_id).await;
@@ -33,7 +33,7 @@ impl RoomEventFetcher {
                 return Ok(config);
             }
             Err(guard) => {
-                let event = room.event(event_id).await?;
+                let event = room.event(event_id, None).await?;
 
                 let _ = guard.insert(event.clone());
 

src/conversation/matrix/utils/mod.rs

@@ -9,9 +9,11 @@ use mxlink::matrix_sdk::{
     ruma::events::{
         relation::Thread,
         room::message::{
-            MessageType, OriginalSyncRoomMessageEvent, Relation, RoomMessageEventContent,
+            sanitize::remove_plain_reply_fallback, MessageType, OriginalSyncRoomMessageEvent,
+            Relation, RoomMessageEventContent,
         },
-        AnyMessageLikeEvent, AnyMessageLikeEventContent, AnyTimelineEvent, MessageLikeEvent,
+        AnyMessageLikeEventContent, AnySyncMessageLikeEvent, AnySyncTimelineEvent,
+        SyncMessageLikeEvent,
     },
     Room,
 };
@@ -75,10 +77,10 @@ async fn get_matrix_messages_in_reply_chain_native(
     event_fetcher: &Arc<RoomEventFetcher>,
     room: &Room,
     event_id: OwnedEventId,
-) -> Result<Vec<AnyMessageLikeEvent>, mxlink::matrix_sdk::Error> {
+) -> Result<Vec<AnySyncMessageLikeEvent>, mxlink::matrix_sdk::Error> {
     let mut next_event_id = Some(event_id.clone());
 
-    let mut messages: Vec<AnyMessageLikeEvent> = Vec::new();
+    let mut messages: Vec<AnySyncMessageLikeEvent> = Vec::new();
     let mut handled_event_ids: Vec<OwnedEventId> = Vec::new();
 
     while let Some(next_event_id_in_loop) = next_event_id {
@@ -97,9 +99,9 @@ async fn get_matrix_messages_in_reply_chain_native(
 
         handled_event_ids.push(next_event_id_in_loop.clone());
 
-        let event_deserialized = event.event.deserialize()?;
+        let event_deserialized = event.raw().deserialize()?;
 
-        let AnyTimelineEvent::MessageLike(message_like_event) = event_deserialized else {
+        let AnySyncTimelineEvent::MessageLike(message_like_event) = event_deserialized else {
             tracing::warn!(
                 "Not proceeding past non-MessageLike event: {:?}",
                 event_deserialized
@@ -108,9 +110,9 @@ async fn get_matrix_messages_in_reply_chain_native(
         };
 
         next_event_id = match message_like_event.clone() {
-            AnyMessageLikeEvent::RoomEncrypted(_) => None,
+            AnySyncMessageLikeEvent::RoomEncrypted(_) => None,
-            AnyMessageLikeEvent::RoomMessage(room_message) => {
+            AnySyncMessageLikeEvent::RoomMessage(room_message) => {
-                if let MessageLikeEvent::Original(room_message_original) = room_message {
+                if let SyncMessageLikeEvent::Original(room_message_original) = room_message {
                     match room_message_original.content.relates_to {
                         Some(Relation::Reply { in_reply_to }) => Some(in_reply_to.event_id.clone()),
                         _ => None,
@@ -206,7 +208,7 @@ fn is_message_from_allowed_sender(
 }
 
 pub fn convert_matrix_native_event_to_matrix_message(
-    matrix_native_event: &AnyMessageLikeEvent,
+    matrix_native_event: &AnySyncMessageLikeEvent,
 ) -> Option<MatrixMessage> {
     let Some(content) = matrix_native_event.original_content() else {
         // Redacted message
@@ -229,7 +231,7 @@ pub fn convert_matrix_native_event_to_matrix_message(
     let text = if is_reply {
         // For regular replies, we need to strip the fallback-for-rich replies part.
         // See: https://spec.matrix.org/v1.11/client-server-api/#fallbacks-for-rich-replies
-        strip_rich_reply_fallback_text(&text)
+        remove_plain_reply_fallback(&text).to_owned()
     } else {
         text
     };
@@ -276,13 +278,14 @@ pub fn convert_matrix_native_event_to_matrix_message(
 #[tracing::instrument(name = "determine_interaction_context_for_room_event", skip_all, fields(room_id = room.room_id().as_str(), event_id = current_event.event_id.as_str()))]
 pub async fn determine_interaction_context_for_room_event(
     bot_user_id: &OwnedUserId,
+    bot_display_name: &Option<String>,
     room: &Room,
     current_event: &OriginalSyncRoomMessageEvent,
     current_event_payload: &MessagePayload,
     event_fetcher: &Arc<RoomEventFetcher>,
 ) -> anyhow::Result<Option<InteractionContext>> {
     let current_event_is_mentioning_bot =
-        is_event_mentioning_bot(&current_event.content, bot_user_id);
+        is_event_mentioning_bot(&current_event.content, bot_user_id, bot_display_name);
 
     let Some(relation) = &current_event.content.relates_to else {
         // This is a top-level message. We consider it the start of the thread.
@@ -304,6 +307,7 @@ pub async fn determine_interaction_context_for_room_event(
         Relation::Thread(thread) => {
             determine_interaction_context_for_room_event_related_to_thread(
                 bot_user_id,
+                bot_display_name,
                 room,
                 current_event,
                 event_fetcher,
@@ -328,6 +332,7 @@ pub async fn determine_interaction_context_for_room_event(
 
 async fn determine_interaction_context_for_room_event_related_to_thread(
     bot_user_id: &OwnedUserId,
+    bot_display_name: &Option<String>,
     room: &Room,
     current_event: &OriginalSyncRoomMessageEvent,
     event_fetcher: &Arc<RoomEventFetcher>,
@@ -388,6 +393,7 @@ async fn determine_interaction_context_for_room_event_related_to_thread(
         thread_start_timeline_event,
         thread_info.clone(),
         bot_user_id,
+        bot_display_name,
     )?;
 
     let Some(detailed_message_payload) = thread_start_detailed_message_payload else {
@@ -430,6 +436,7 @@ async fn determine_interaction_context_for_room_event_related_to_reply(
 fn is_event_mentioning_bot(
     event_content: &RoomMessageEventContent,
     bot_user_id: &OwnedUserId,
+    bot_display_name: &Option<String>,
 ) -> bool {
     if let Some(mentions) = &event_content.mentions {
         mentions
@@ -444,39 +451,18 @@ fn is_event_mentioning_bot(
         // As of 2024-10-03, at least Element iOS does not support the new Mentions specification
         // and is still quite widespread.
         //
-        // It may be even better to match not only against the MXID, but also against the bot's
-        // room-specific display name.
-        //
         // We may consider dropping this string-matching behavior altogether in the future,
         // so improving this compatibility block is not a high priority.
-        event_content.body().contains(bot_user_id.as_str())
+        if event_content.body().contains(bot_user_id.as_str()) {
-    }
+            return true;
-}
-
-/// Strips the rich reply fallback text from the given text.
-/// See: https://spec.matrix.org/v1.11/client-server-api/#fallbacks-for-rich-replies
-///
-/// Example:
-/// ```rust,ignore
-/// let text = "> <@admin:example.com> What's the difference between Matrix and XMPP?\n\nAnswer me";
-/// let stripped_text = strip_rich_reply_fallback_text(text);
-/// assert_eq!(stripped_text, "Answer me");
-/// ```
-fn strip_rich_reply_fallback_text(text: &str) -> String {
-    let lines = text.lines();
-    let mut stripped_lines = Vec::new();
-    let mut encountered_non_prefix = false;
-
-    for line in lines {
-        if !encountered_non_prefix && line.starts_with("> ") {
-            continue;
-        } else {
-            encountered_non_prefix = true;
-            stripped_lines.push(line);
         }
-    }
 
-    stripped_lines.join("\n").trim().to_owned()
+        if let Some(bot_display_name) = bot_display_name {
+            return event_content.body().contains(bot_display_name);
+        }
+
+        false
+    }
 }
 
 fn timeline_event_to_detailed_message_payload(
@@ -484,8 +470,9 @@ fn timeline_event_to_detailed_message_payload(
     timeline_event: TimelineEvent,
     thread_info: ThreadInfo,
     bot_user_id: &OwnedUserId,
+    bot_display_name: &Option<String>,
 ) -> anyhow::Result<Option<DetailedMessagePayload>> {
-    let timeline_event_deserialized = match timeline_event.event.deserialize() {
+    let timeline_event_deserialized = match timeline_event.raw().deserialize() {
         Ok(value) => value,
         Err(err) => {
             return Err(anyhow::format_err!(
@@ -496,7 +483,7 @@ fn timeline_event_to_detailed_message_payload(
         }
     };
 
-    let AnyTimelineEvent::MessageLike(thread_start_message_like_event) =
+    let AnySyncTimelineEvent::MessageLike(thread_start_message_like_event) =
         timeline_event_deserialized
     else {
         tracing::trace!(
@@ -507,7 +494,7 @@ fn timeline_event_to_detailed_message_payload(
     };
 
     let (is_mentioning_bot, message_payload) = match thread_start_message_like_event {
-        AnyMessageLikeEvent::RoomEncrypted(room_message) => {
+        AnySyncMessageLikeEvent::RoomEncrypted(room_message) => {
             tracing::warn!(
                 "Could not inspect event {} because it failed to decrypt: {:?}",
                 timeline_event_id.clone(),
@@ -522,8 +509,8 @@ fn timeline_event_to_detailed_message_payload(
                 MessagePayload::Encrypted(thread_info.clone()),
             )
         }
-        AnyMessageLikeEvent::RoomMessage(room_message) => {
+        AnySyncMessageLikeEvent::RoomMessage(room_message) => {
-            if let MessageLikeEvent::Original(room_message_original) = room_message {
+            if let SyncMessageLikeEvent::Original(room_message_original) = room_message {
                 let room_message_payload: Result<MessagePayload, String> =
                     room_message_original.content.msgtype.clone().try_into();
 
@@ -535,8 +522,11 @@ fn timeline_event_to_detailed_message_payload(
                     return Ok(None);
                 };
 
-                let is_mentioning_bot =
+                let is_mentioning_bot = is_event_mentioning_bot(
-                    is_event_mentioning_bot(&room_message_original.content, bot_user_id);
+                    &room_message_original.content,
+                    bot_user_id,
+                    bot_display_name,
+                );
 
                 (is_mentioning_bot, room_message_payload)
             } else {

src/conversation/matrix/utils/tests.rs

@@ -296,13 +296,6 @@ async fn process_matrix_messages() {
     }
 }
 
-#[test]
-fn strip_rich_reply_fallback_text() {
-    let text = "> <@admin:example.com> What's the difference between Matrix and XMPP?\n\nAnswer me";
-    let stripped_text = super::strip_rich_reply_fallback_text(text);
-    assert_eq!(stripped_text, "Answer me");
-}
-
 #[test]
 fn create_list_of_bot_user_prefixes_to_strip() {
     let bot_user_id =

src/entity/message_context.rs

@@ -15,6 +15,8 @@ pub struct MessageContext {
     admin_whitelist_regexes: Vec<regex::Regex>,
     trigger_event_info: TriggerEventInfo,
     thread_info: ThreadInfo,
+
+    bot_display_name: Option<String>,
 }
 
 impl MessageContext {
@@ -31,9 +33,20 @@ impl MessageContext {
             admin_whitelist_regexes,
             trigger_event_info,
             thread_info,
+
+            bot_display_name: None,
         }
     }
 
+    pub fn with_bot_display_name(mut self, value: Option<String>) -> Self {
+        self.bot_display_name = value;
+        self
+    }
+
+    pub fn bot_display_name(&self) -> &Option<String> {
+        &self.bot_display_name
+    }
+
     pub fn room(&self) -> &Room {
         &self.room
     }