Release 1.8.2

No need to have both sample configs now.

Fixes https://github.com/etkecc/baibot/issues/57

CHANGELOG.md

@@ -1,3 +1,64 @@
+# (2025-11-20) Version 1.8.2
+
+- (**Internal Improvement**) Dependency and compiler updates (Rust 1.89.0 -> 1.91.1).
+
+# (2025-09-12) Version 1.8.1
+
+- (**Internal Improvement**) Dependency updates.
+
+# (2025-09-08) Version 1.8.0
+
+- (**Internal Improvement**) Upgrade [mxlink](https://crates.io/crates/mxlink) (1.9.0 -> 1.10.0) and [matrix-sdk](https://crates.io/crates/matrix-sdk) (0.13.0 -> 0.14.0)
+
+- (**Internal Improvement**) Upgrade [Rust](https://www.rust-lang.org/) (1.88.0 -> 1.89.0)
+
+- (**Internal Improvement**) Upgrade Debian base for container images (12/bookworm -> 13/trixie)
+
+# (2025-07-11) Version 1.7.6
+
+- (**Internal Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.9.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.13.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.13.0), which contains fixes for some security vulnerabilities)
+
+# (2025-06-10) Version 1.7.5
+
+- (**Internal Improvement**) Dependency and compiler updates (Rust 1.86 -> 1.86).
+
+# (2025-06-10) Version 1.7.4
+
+- (**Internal Improvement**) Dependency updates.
+
+# (2025-06-10) Version 1.7.3
+
+- (**Internal Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.8.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.12.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.12.0), which contains fixes for important security vulnerabilities)
+
+# (2025-05-11) Version 1.7.2
+
+- (**Bugfix**) Allow `image_generation.size` configuration value for OpenAI to be `null` to allow the model to choose the size automatically and default to that
+
+# (2025-05-11) Version 1.7.1
+
+- (**Bugfix**) Fix lack of documentation for the new [image-editing](./docs/features.md#-image-editing) feature in the `!bai usage` command's output
+
+# (2025-05-10) Version 1.7.0
+
+- (**Feature**) Add vision support to the OpenAI and Anthropic providers. You can now mix text and images in your conversations - fixes [issue #5](https://github.com/etkecc/baibot/issues/5)
+
+- (**Feature**) Add [image-editing](./docs/features.md#-image-editing) support to the OpenAI provider
+
+- (**Improvement**) Add compatibility with OpenAI's `gpt-image-1` model - fixes [issue #40](https://github.com/etkecc/baibot/issues/40)
+
+- (**Change**) Rework [image-creation](./docs/features.md#-image-creation) to avoid command conflicts with [image-editing](./docs/features.md#-image-editing). The image-creation command syntax is now `!bai image create <prompt>` (previously: `!bai image <prompt>`).
+
+- (**Internal Improvement**) Dependency and compiler updates
+
+> [!WARNING]
+> Unlike other releases, this release is not published to [crates.io](https://crates.io), because it relies on multiple library forks (`async-openai` and `anthropic-rs`) sourced from Github.
+
+
+# (2025-04-12) Version 1.6.0
+
+- (**Internal Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.7.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.11.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.11.0))
+
+
 # (2025-03-31) Version 1.5.1
 
 - (**Internal Improvement**) Dependency updates

Cargo.toml

@@ -7,7 +7,7 @@ license = "AGPL-3.0-or-later"
 readme = "README.md"
 keywords = ["matrix", "chat", "bot", "AI", "LLM"]
 include = ["/etc/assets/baibot-torso-768.png", "/src", "/README.md", "/CHANGELOG.md", "/LICENSE"]
-version = "1.5.1"
+version = "1.8.2"
 edition = "2024"
 
 [lib]
@@ -15,25 +15,25 @@ name = "baibot"
 path = "src/lib.rs"
 
 [dependencies]
-anthropic = "=0.0.8"
+anthropic = { git = "https://github.com/etkecc/anthropic-rs.git", branch = "fix-content-block-image" }
 anyhow = "1.0.*"
-async-openai = "0.28.*"
+async-openai = { git = "https://github.com/etkecc/async-openai", branch = "async-openai-v0.28.1-patched" }
 base64 = "0.22.*"
 chrono = { version = "0.4.*", default-features = false, features = ["std", "now"] }
 # We'd rather not depend on this, but we cannot use the ruma-events EventContent macro without it.
 # We add the `native-tls` feature, because of https://github.com/etkecc/rust-mxlink/issues/1
-matrix-sdk = { version = "0.10.0", default-features = false, features = ["native-tls"] }
+matrix-sdk = { version = "0.14.0", default-features = false, features = ["native-tls"] }
 mxidwc = "1.0.*"
-mxlink = ">=1.6.0"
+mxlink = ">=1.10.0"
 etke_openai_api_rust = "0.1.*"
 quick_cache = "0.6.*"
-regex = "1.11.*"
+regex = "1.12.*"
 serde = { version = "1.0.*", features = ["derive"], default-features = false }
 serde_json = "1.0.*"
 serde_yaml = "0.9.*"
-tempfile = "3.19.*"
-tiktoken-rs = { version = "0.6.*", features = ["async-openai"] }
-tokio = { version = "1.44.*", features = ["rt", "rt-multi-thread", "macros"] }
+tempfile = "3.23.*"
+tiktoken-rs = { version = "0.9.*", default-features = false }
+tokio = { version = "1.48.*", features = ["rt", "rt-multi-thread", "macros"] }
 tracing = "0.1.*"
 tracing-subscriber = { version = "0.3.*", features = ["env-filter"] }
 url = "2.5.*"

Dockerfile

@@ -4,7 +4,7 @@
 #                                     #
 #######################################
 
-FROM docker.io/rust:1.85.1-slim-bookworm AS build
+FROM docker.io/rust:1.91.1-slim-trixie AS build
 
 RUN apt-get update && apt-get install -y build-essential pkg-config libssl-dev libsqlite3-dev
 
@@ -39,7 +39,7 @@ RUN --mount=type=cache,target=/target,sharing=locked \
 #                                     #
 #######################################
 
-FROM docker.io/debian:bookworm-slim
+FROM docker.io/debian:trixie-slim
 
 RUN apt-get update && apt-get install -y ca-certificates sqlite3 && \
     apt-get clean && \

Dockerfile.ci

@@ -4,7 +4,7 @@
 #                                     #
 #######################################
 
-FROM docker.io/rust:1.85.1-slim-bookworm AS build
+FROM docker.io/rust:1.91.1-slim-trixie AS build
 
 RUN apt-get update && apt-get install -y build-essential pkg-config libssl-dev libsqlite3-dev
 
@@ -20,7 +20,7 @@ RUN cargo build --release
 #                                     #
 #######################################
 
-FROM docker.io/debian:bookworm-slim
+FROM docker.io/debian:trixie-slim
 
 RUN apt-get update && apt-get install -y ca-certificates sqlite3 && \
     apt-get clean && \

README.md

@@ -17,10 +17,10 @@ It's influenced by [chaz](https://github.com/arcuru/chaz), but does **not** use
 
 - Supports **different use purposes** (depending on the [☁️ provider](./docs/providers.md) & model):
 
-  - [💬 text-generation](./docs/features.md#-text-generation): communicating with you via text
+  - [💬 text-generation](./docs/features.md#-text-generation): communicating with you via text (though certain models may "see" images as well)
   - [🦻 speech-to-text](./docs/features.md#-speech-to-text): turning your voice messages into text
   - [🗣️ text-to-speech](./docs/features.md#%EF%B8%8F-text-to-speech): turning bot or users text messages into voice messages
-  - [🖌️ image-generation](./docs/features.md#%EF%B8%8F-image-generation): generating images based on instructions
+  - [🖌️ image-generation](./docs/features.md#image-generation): creating and editing images based on instructions
 
 - 🪄 Supports [seamless voice interaction](./docs/features.md#seamless-voice-interaction) (turning user voice messages into text, answering in text, then turning that text back into voice)
 

docs/agents.md

@@ -35,7 +35,7 @@ Depending on where the agent is defined (within a room, globally, or [statically
 
 When creating an agent, you will be given some sample [YAML](https://en.wikipedia.org/wiki/YAML) configuration which you can use to customize the agent's behavior.
 
-This configuration varies depending on the [☁️ provider](./providers.md) used and the capabilities of the agent. Based on the configuration keys you pass, certain features will be enabled or disabled. For example, if you skip the `image_generation` key for an [OpenAI](./providers.md#openai) agent, it won't be able to generate images (see [🖌️ Image Generation](./features.md#-image-generation)).
+This configuration varies depending on the [☁️ provider](./providers.md) used and the capabilities of the agent. Based on the configuration keys you pass, certain features will be enabled or disabled. For example, if you skip the `image_generation` key for an [OpenAI](./providers.md#openai) agent, it won't be able to generate images (see [🖌️ Image Creation](./features.md#-image-creation), [🎨 Image Editing](./features.md#-image-editing), [🫵 Sticker Creation](./features.md#-sticker-creation)).
 
 After making your modifications to the sample YAML, you submit it back to the bot and the new agent will be created.
 

docs/configuration/README.md

@@ -40,7 +40,7 @@ You can adjust the following settings per room and/or globally:
 - [💬 Text Generation](text-generation.md)
 - [🦻 Speech-to-Text](speech-to-text.md)
 - [🗣️ Text-to-Speech](text-to-speech.md)
-- [🖌️ Image Generation](image-generation.md)
+- [🖌️ Image Creation](image-generation.md)
 - [🤝 Handlers](handlers.md)
 
 Refer to the bot's help messages (as a response to a `!bai config` help command) for the most up-to-date information on what Room Settings can be configured.

docs/configuration/handlers.md

@@ -8,10 +8,10 @@ You can also use **different models within the same room** (e.g. [💬 text-gene
 
 The bot supports the following use-purposes:
 
-- [💬 text-generation](../features.md#-text-generation): communicating with you via text
+- [💬 text-generation](../features.md#-text-generation): communicating with you via text (though certain models may "see" images as well)
 - [🦻 speech-to-text](../features.md#-speech-to-text): turning your voice messages into text
 - [🗣️ text-to-speech](../features.md#️-text-to-speech): turning bot or users text messages into voice messages
-- [🖌️ image-generation](../features.md#-image-generation): generating images based on instructions
+- [🖌️ image-generation](../features.md#image-generation): generating images based on instructions
 
 In a given room, each different purpose can be served by a different [provider](../providers.md) and model. This combination of provider and model configuration is called an [🤖 agent](../agents.md). Each purpose can be served by a different **handler** agent.
 

docs/configuration/image-generation.md

@@ -1,9 +1,11 @@
 
-## 🖌️ Image Generation
+## Image Generation
 
-The Image Generation feature is not configurable at this moment.
+The Image Creation and Image Editing features are not configurable at this moment.
 
 You may also wish to see:
 
-- [🌟 Features / 🖌️ Image Generation](../features.md#-image-generation) for a higher-level introduction to the Image Generation features
-- [📖 Usage / 🖌️ Image Generation](../usage.md#-image-generation) section for more details on how to use the bot for Image Generation in a room
+- [🌟 Features / Image Generation / 🖌️ Image Creation](../features.md#-image-creation) for a higher-level introduction to the Image Creation features
+- [🌟 Features / Image Generation / 🎨 Image Editing](../features.md#-image-editing) for a higher-level introduction to the Image Editing features
+- [📖 Usage / Image Generation / 🖌️ Creating Images](../usage.md#-creating-images) section for more details on how to use the bot for Image Creation in a room
+- [📖 Usage / Image Generation / 🎨 Editing images](../usage.md#-editing-images) section for more details on how to use the bot for Image Editing in a room

docs/development.md

@@ -93,7 +93,7 @@ For getting started most quickly (and locally), we recommend using [LocalAI](#lo
 
 **Ollama is most lightweight** (~2GB for the container image + ~1.6GB for the model), but supports only [💬 text-generation](./features.md#-text-generation).
 
-**LocalAI requires 4x more disk space** (~6GB for the container image + ~12GB for the models), but supports [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text) and [🖼️ image-generation](./features.md#️-image-generation).
+**LocalAI requires 4x more disk space** (~6GB for the container image + ~12GB for the models), but supports [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text) and [🖼️ image-generation](./features.md#️-image-creation).
 
 **OpenAI supports all of these capabilities** as well and does not require powerful hardware or lots of disk space. However, it requires signup and an API key.
 

docs/features.md

@@ -8,7 +8,7 @@ You can also use **different models within the same room** (e.g. [💬 text-gene
 
 The bot supports the following use-purposes:
 
-- [💬 text-generation](#-text-generation): communicating with you via text
+- [💬 text-generation](#-text-generation): communicating with you via text (though certain models may "see" images as well)
 - [🦻 speech-to-text](#-speech-to-text): turning your voice messages into text
 - [🗣️ text-to-speech](#%EF%B8%8F-text-to-speech): turning bot or users text messages into voice messages
 - [🖌️ image-generation](#%EF%B8%8F-image-generation): generating images based on instructions
@@ -22,10 +22,12 @@ For more information about configuring handlers, see the [🤝 Handlers / Config
 
 ### 💬 Text Generation
 
-Text Generation is the bot's ability to **respond to users' text messages with text**.
+Text Generation is the bot's ability to **respond to users' messages with text**.
 
 ![Screenshot of Text Generation - a user sends a message and the bot replies in a new conversation thread](./screenshots/text-generation.webp)
 
+Some models also support vision, so you may be able to mix text and images in the same conversation.
+
 In multi-user (group) rooms, to avoid disturbing the normal conversation between people, the bot is auto-configured to only respond to messages starting with the command prefix (`!bai`) or direct mentions via the [💬 Text Generation / 🗟 Prefix Requirement Type](./configuration/text-generation.md#-prefix-requirement-type) setting.
 
 Normally, the bot only responds to allowed [👥 Users](./access.md#-users). In certain cases, it's useful for an allowed user to provoke the bot to respond even in foreign threads or reply chains. You can learn more about this feature in the [On-demand involvement](./features.md#on-demand-involvement) section below.
@@ -139,26 +141,42 @@ To operate in this mode, you can:
 - optionally adjust [🦻 Speech-to-Text / 🪄 Message Type for non-threaded only-transcribed messages](./configuration/speech-to-text.md#-message-type-for-non-threaded-only-transcribed-messages), if you'd like to bot to send messages of type `notice` (for better compatibility with other bots in the room) instead of sending regular `text` messages (default)
 
 
-### 🖌️ Image Generation
+### Image Generation
 
-Image generation is the bot's ability to **generate images** based on text prompts.
+#### 🖌️ Image Creation
 
-See a [🖼️ Screenshot of the Image Generation feature](./screenshots/image-generation.webp).
+Image creation is the bot's ability to **create images** based on text prompts.
+
+See a [🖼️ Screenshot of the Image Creation feature](./screenshots/image-creation.webp).
 
 You may also wish to see:
 
 - [🛠️ Configuration / 🖌️ Image Generation](./configuration/image-generation.md) for configuration options related to Image Generation
-- [📖 Usage / 🖌️ Image Generation](./usage.md#-image-generation) section for more details on how to use the bot for Image Generation in a room
-- [🫵 Sticker Generation](#-sticker-generation) - a special case of Image Generation
+- [📖 Usage / Image Generation / 🖌️ Creating Images](./usage.md#-creating-images) section for more details on how to use the bot for Image Creation in a room
+- [🖌️ Image Editing](#️-image-editing) - another image generation feature
+- [🫵 Sticker Creation](#-sticker-creation) - a special case of Image Creation
 
 
-### 🫵 Sticker Generation
+#### 🎨 Image Editing
 
-Sticker generation is the bot's ability to **generate sticker** images based on text prompts. It's a special case of [🖌️ Image Generation](#️-image-generation).
+Image editing is the bot's ability to **edit images** based on a prompt and one or more existing images.
 
-See a [🖼️ Screenshot of the Sticker Generation feature](./screenshots/sticker-generation.webp).
+See a [🖼️ Screenshot of the Image Editing feature (manipulating a single image)](./screenshots/image-editing-single-image.webp) and a [🖼️ Screenshot of the Image Editing feature (manipulating multiple images)](./screenshots/image-editing-multiple-images.webp).
 
-See [📖 Usage / 🖌️ Image Generation / Generating Stickers](./usage.md#generating-stickers) for details.
+You may also wish to see:
+
+- [🛠️ Configuration / 🖌️ Image Generation](./configuration/image-generation.md) for configuration options related to Image Generation
+- [📖 Usage / Image Generation / 🎨 Editing images](./usage.md#-editing-images) section for more details on how to use the bot for Image Editing in a room
+- [🖌️ Image Creation](#️-image-creation) - another image generation feature
+
+
+#### 🫵 Sticker Creation
+
+Sticker generation is the bot's ability to **generate sticker** images based on text prompts. It's a special case of [🖌️ Image Creation](#️-image-creation).
+
+See a [🖼️ Screenshot of the Sticker Creation feature](./screenshots/sticker-generation.webp).
+
+See [📖 Usage / Image Generation / 🫵 Creating Stickers](./usage.md#-creating-stickers) for details.
 
 
 ### 🔒 Encryption

docs/installation.md

@@ -53,6 +53,7 @@ CONTAINER_IMAGE_NAME=ghcr.io/etkecc/baibot:v1.0.0
   --env BAIBOT_PERSISTENCE_DATA_DIR_PATH=/data \
   --mount type=bind,src=/path/to/config.yml,dst=/app/config.yml,ro \
   --mount type=bind,src=/path/to/data,dst=/data \
+  --tmpfs=/tmp:rw,noexec,nosuid,size=1024m \
   $CONTAINER_IMAGE_NAME
 ```
 

docs/providers.md

@@ -23,7 +23,7 @@ The list of supported providers is below.
 
 ### How to choose a provider
 
-If you're not sure which provider to start with, **we recommend [OpenAI](#openai)** as it's the most popular and has the **widest range of capabilities**: [💬 text-generation](./features.md#-text-generation), [🖌️ image-generation](./features.md#️-image-generation), [🦻 speech-to-text](./features.md#-speech-to-text), [🗣️ text-to-speech](./features.md#️-text-to-speech).
+If you're not sure which provider to start with, **we recommend [OpenAI](#openai)** as it's the most popular and has the **widest range of capabilities**: [💬 text-generation](./features.md#-text-generation) (no vision), [🖌️ image-generation](./features.md#️image-generation), [🦻 speech-to-text](./features.md#-speech-to-text), [🗣️ text-to-speech](./features.md#️-text-to-speech).
 
 You don't need to choose just one though. The bot supports [mixing & matching models](./features.md#-mixing--matching-models), so you can use multiple providers at the same time.
 
@@ -47,7 +47,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
 
 - 🆔 Identifier: `anthropic`
 - 🔗 Links: [🏠 Home page](https://www.anthropic.com/), [🌐 Wiki](https://en.wikipedia.org/wiki/Anthropic), [👤 Sign up](https://console.anthropic.com/), [📋 Models list](https://docs.anthropic.com/en/docs/about-claude/models)
-- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation)
+- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (incl. vision)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local anthropic my-anthropic-agent`
   - create a global agent: `!bai agent create-global anthropic my-anthropic-agent`
@@ -61,7 +61,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
 
 - 🆔 Identifier: `groq`
 - 🔗 Links: [🏠 Home page](https://groq.com/), [🌐 Wiki](https://en.wikipedia.org/wiki/Groq), [👤 Sign up](https://console.groq.com/login), [📋 Models list](https://console.groq.com/docs/models)
-- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation), [🦻 speech-to-text](./features.md#-speech-to-text)
+- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision), [🦻 speech-to-text](./features.md#-speech-to-text)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local groq my-groq-agent`
   - create a global agent: `!bai agent create-global groq my-groq-agent`
@@ -75,7 +75,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
 
 - 🆔 Identifier: `localai`
 - 🔗 Links: [🏠 Home page](https://localai.io/), [📋 Models list](https://localai.io/gallery.html)
-- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
+- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local localai my-localai-agent`
   - create a global agent: `!bai agent create-global localai my-localai-agent`
@@ -89,7 +89,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
 
 - 🆔 Identifier: `mistral`
 - 🔗 Links: [🏠 Home page](https://mistral.ai/), [🌐 Wiki](https://en.wikipedia.org/wiki/Mistral_AI), [👤 Sign up](https://auth.mistral.ai/ui/registration), [📋 Models list](https://docs.mistral.ai/getting-started/models/)
-- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation)
+- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local mistral my-mistral-agent`
   - create a global agent: `!bai agent create-global mistral my-mistral-agent`
@@ -103,7 +103,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
 
 - 🆔 Identifier: `ollama`
 - 🔗 Links: [🏠 Home page](https://ollama.com/), [📋 Models list](https://ollama.com/library)
-- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation)
+- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local ollama my-ollama-agent`
   - create a global agent: `!bai agent create-global ollama my-ollama-agent`
@@ -120,15 +120,12 @@ For services which are not fully compatible with the OpenAI API, consider using
 
 - 🆔 Identifier: `openai`
 - 🔗 Links: [🏠 Home page](https://openai.com/), [🌐 Wiki](https://en.wikipedia.org/wiki/OpenAI), [👤 Sign up](https://platform.openai.com/signup), [📋 Models list](https://platform.openai.com/docs/models)
-- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-generation), [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
+- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-creation), [💬 text-generation](./features.md#-text-generation) (incl. vision), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local openai my-openai-agent`
   - create a global agent: `!bai agent create-global openai my-openai-agent`
 
-💡 When creating an agent, the bot will show you an up-to-date sample configuration for this provider which:
-
-- in the general case looks [like this](./sample-provider-configs/openai.yml)
-- for the [o1](https://platform.openai.com/docs/models/o1) models needs to look [like this](./sample-provider-configs/openai-o1.yml)
+💡 When creating an agent, the bot will show you an up-to-date sample configuration for this provider which looks [like this](./sample-provider-configs/openai.yml).
 
 
 ### OpenAI Compatible
@@ -140,7 +137,7 @@ Some of these popular services already have **shortcut** providers (leading to t
 This provider is just as featureful as the [OpenAI](#openai) provider, but is more compatible with services which do not fully adhere to the [OpenAI API spec](https://github.com/openai/openai-openapi/).
 
 - 🆔 Identifier: `openai-compatible`
-- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-generation), [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
+- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-creation), [💬 text-generation](./features.md#-text-generation) (no vision), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local openai-compatible my-openai-compatible-agent`
   - create a global agent: `!bai agent create-global openai-compatible my-openai-compatible-agent`
@@ -154,7 +151,7 @@ This provider is just as featureful as the [OpenAI](#openai) provider, but is mo
 
 - 🆔 Identifier: `openrouter`
 - 🔗 Links: [🏠 Home page](https://openrouter.ai/), [👤 Sign up](https://openrouter.ai/), [📋 Models list](https://openrouter.ai/models)
-- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation)
+- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local openrouter my-openrouter-agent`
   - create a global agent: `!bai agent create-global openrouter my-openrouter-agent`
@@ -168,7 +165,7 @@ This provider is just as featureful as the [OpenAI](#openai) provider, but is mo
 
 - 🆔 Identifier: `together-ai`
 - 🔗 Links: [🏠 Home page](https://www.together.ai/), [👤 Sign up](https://api.together.ai/signup), [📋 Models list](https://api.together.xyz/models)
-- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation)
+- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local together-ai my-together-ai-agent`
   - create a global agent: `!bai agent create-global together-ai my-together-ai-agent`

docs/sample-provider-configs/openai-o1.yml

@@ -1,24 +0,0 @@
-base_url: https://api.openai.com/v1
-api_key: YOUR_API_KEY_HERE
-text_generation:
-  model_id: o1-mini
-  # o1 models do not support a system prompt
-  prompt: null
-  temperature: 1.0
-  # o1 models do not support max_response_tokens.
-  # They use `max_completion_tokens` as an alternative
-  max_response_tokens: null
-  max_completion_tokens: 16384
-  max_context_tokens: 128000
-speech_to_text:
-  model_id: whisper-1
-text_to_speech:
-  model_id: tts-1-hd
-  voice: onyx
-  speed: 1.0
-  response_format: opus
-image_generation:
-  model_id: dall-e-3
-  style: vivid
-  size: 1024x1024
-  quality: standard

docs/sample-provider-configs/openai.yml

@@ -1,11 +1,14 @@
 base_url: https://api.openai.com/v1
 api_key: YOUR_API_KEY_HERE
 text_generation:
-  model_id: gpt-4o
+  model_id: gpt-5.1
   prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
   temperature: 1.0
-  max_response_tokens: 16384
-  max_context_tokens: 128000
+  # Reasoning models need to use `max_completion_tokens` instead of `max_response_tokens`.
+  # If you're dealing with a non-reasoning model, specify `max_response_tokens` and unset `max_completion_tokens`.
+  max_response_tokens: null
+  max_completion_tokens: 128000
+  max_context_tokens: 400000
 speech_to_text:
   model_id: whisper-1
 text_to_speech:
@@ -14,7 +17,7 @@ text_to_speech:
   speed: 1.0
   response_format: opus
 image_generation:
-  model_id: dall-e-3
-  style: vivid
-  size: 1024x1024
-  quality: standard
+  model_id: gpt-image-1
+  style: null
+  size: null
+  quality: null

docs/usage.md

@@ -11,6 +11,8 @@ This is related to the [💬 Text Generation](./features.md#-text-generation) fe
 
 If there's a text-generation handler agent configured, the bot **may** respond to messages sent in the room.
 
+Some models also support vision, so you may be able to mix text and images in the same conversation.
+
 See screenshots of:
 
 - 🖼️ [the default Text Generation flow](./screenshots/text-generation.webp) in 1:1 rooms
@@ -64,34 +66,48 @@ The speech-to-text feature triggers automatically by default, but can be adjuste
 If all your messages are in the same language, you can improve accuracy & latency by configuring the language (see [🦻 Speech-to-Text / 🔤 Language](./configuration/speech-to-text.md#-language)).
 
 
-### 🖌️ Image Generation
-
-This is related to the [🖌️ Image Generation](./features.md#️-image-generation) feature.
+### Image Generation
 
 This feature is not configurable at the moment. The configuration (size, quality, style) specified at the [🤖 agent](./agents.md) level will be used.
 
+Capabilities depend on the [☁️ provider](./providers.md) and model used.
 
-#### Generating images
 
-Simply send a command like `!bai image A beautiful sunset over the ocean` and the bot will start a threaded conversation and post an image based on your prompt.
+#### 🖌️ Creating images
 
-See a [🖼️ Screenshot of the Image Generation feature](./screenshots/image-generation.webp).
+Simply send a command like `!bai image create A beautiful sunset over the ocean` and the bot will start a threaded conversation and post an image based on your prompt.
 
-You can then, respond in the same message thread with:
+See a [🖼️ Screenshot of the Image Creation feature](./screenshots/image-creation.webp).
+
+You can then respond in the same message thread with:
 
 - more messages, to add more criteria to your prompt.
 - a message saying `again`, to generate one more image with the current prompt.
 
 
-#### Generating stickers
+#### 🎨 Editing images
 
-A variation of [generating images](#generating-images) is to generate "sticker images".
+Simply send a command like `!bai image edit Turn the following image into an anime-style drawing` and the bot will start a threaded conversation asking for more details.
 
-See a [🖼️ Screenshot of the Sticker Generation feature](./screenshots/sticker-generation.webp).
+See a [🖼️ Screenshot of the Image Editing feature (manipulating a single image)](./screenshots/image-editing-single-image.webp) and a [🖼️ Screenshot of the Image Editing feature (manipulating multiple images)](./screenshots/image-editing-multiple-images.webp).
 
-To generate a sticker, send a command like `!bai sticker A huge ramen bowl with lots of chashu and a mountain of beansprouts on top`.
+You can then respond in the same message thread with:
 
-The difference from [generating images](#generating-images) is that the bot will:
+- more messages, to add more criteria to your prompt.
+- one or more images, to provide the images that the bot will operate on.
+- a message saying `go`, to start the image generation process.
+- a message saying `again`, to prompt the bot to generate one more image edit with the current prompt.
+
+
+#### 🫵 Creating stickers
+
+A variation of [creating images](#creating-images) is creating "sticker images".
+
+See a [🖼️ Screenshot of the Sticker Creation feature](./screenshots/sticker-generation.webp).
+
+To create a sticker, send a command like `!bai sticker A huge ramen bowl with lots of chashu and a mountain of beansprouts on top`.
+
+The difference from [creating images](#creating-images) is that the bot will:
 
 - generate a smaller-resolution image (currently hardcoded to `256x256`) - smaller/quicker, but still good enough for a sticker
 - potentially switch to a different (cheaper or otherwise more suitable) model, if available

etc/app/config.yml.dist

@@ -76,13 +76,14 @@ agents:
     #     base_url: https://api.openai.com/v1
     #     api_key: ""
     #     text_generation:
-    #       model_id: gpt-4o
+    #       model_id: gpt-5.1
     #       prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
     #       temperature: 1.0
-    #       max_response_tokens: 16384
     #       # Reasoning models need to use `max_completion_tokens` instead of `max_response_tokens`.
-    #       max_completion_tokens: ~
-    #       max_context_tokens: 128000
+    #       # If you're dealing with a non-reasoning model, specify `max_response_tokens` and unset `max_completion_tokens`.
+    #       max_response_tokens: null
+    #       max_completion_tokens: 128000
+    #       max_context_tokens: 400000
     #     speech_to_text:
     #       model_id: whisper-1
     #     text_to_speech:
@@ -91,10 +92,10 @@ agents:
     #       speed: 1.0
     #       response_format: opus
     #     image_generation:
-    #       model_id: dall-e-3
-    #       style: vivid
-    #       size: 1024x1024
-    #       quality: standard
+    #       model_id: gpt-image-1
+    #       style: null
+    #       size: null
+    #       quality: null
     #
     # - id: localai
     #   provider: localai

etc/services/core/compose.yml

@@ -1,6 +1,6 @@
 services:
   postgres:
-    image: docker.io/postgres:16.8-alpine
+    image: docker.io/postgres:18.1-alpine
     user: ${UID}:${GID}
     restart: unless-stopped
     environment:
@@ -8,12 +8,13 @@ services:
      POSTGRES_PASSWORD: synapse-password
      POSTGRES_DB: homeserver
      POSTGRES_INITDB_ARGS: --lc-collate C --lc-ctype C --encoding UTF8
+     PGDATA: /data
     volumes:
-    - ./postgres:/var/lib/postgresql/data
+    - ./postgres:/data
     - /etc/passwd:/etc/passwd:ro
 
   synapse:
-    image: ghcr.io/element-hq/synapse:v1.127.1
+    image: ghcr.io/element-hq/synapse:v1.142.1
     user: "${UID}:${GID}"
     restart: unless-stopped
     entrypoint: python
@@ -26,7 +27,7 @@ services:
     - ./synapse/media-store:/media-store
 
   element-web:
-    image: ghcr.io/element-hq/element-web:v1.11.96
+    image: ghcr.io/element-hq/element-web:v1.12.4
     user: "${UID}:${GID}"
     restart: unless-stopped
     environment:

etc/services/ollama/compose.yml

@@ -1,6 +1,6 @@
 services:
   ollama:
-    image: docker.io/ollama/ollama:0.6.3
+    image: docker.io/ollama/ollama:0.13.0
     restart: unless-stopped
     ports:
       - "${SERVICE_OLLAMA_BIND_PORT_HTTP}:11434"

src/agent/provider/anthropic/controller.rs

@@ -7,13 +7,15 @@ use anthropic::types::ContentBlock;
 use super::super::ControllerTrait;
 use crate::agent::AgentPurpose;
 use crate::agent::provider::entity::{
-    ImageGenerationResult, PingResult, TextGenerationParams, TextGenerationResult,
-    TextToSpeechParams, TextToSpeechResult,
+    ImageEditResult, ImageGenerationResult, ImageSource, PingResult, TextGenerationParams,
+    TextGenerationResult, TextToSpeechParams, TextToSpeechResult,
+};
+use crate::agent::provider::{
+    ImageEditParams, ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
 };
-use crate::agent::provider::{ImageGenerationParams, SpeechToTextParams, SpeechToTextResult};
 use crate::conversation::llm::{
     Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
-    shorten_messages_list_to_context_size,
+    MessageContent as LLMMessageContent, shorten_messages_list_to_context_size,
 };
 use crate::strings;
 
@@ -69,7 +71,7 @@ impl ControllerTrait for Controller {
 
         let messages = vec![LLMMessage {
             author: LLMAuthor::User,
-            message_text: "Hello!".to_string(),
+            content: LLMMessageContent::Text("Hello!".to_string()),
             timestamp: chrono::Utc::now(),
         }];
 
@@ -106,7 +108,7 @@ impl ControllerTrait for Controller {
         } else {
             Some(LLMMessage {
                 author: LLMAuthor::Prompt,
-                message_text: prompt_text,
+                content: LLMMessageContent::Text(prompt_text),
                 timestamp: chrono::Utc::now(),
             })
         };
@@ -145,7 +147,9 @@ impl ControllerTrait for Controller {
             .unwrap_or(text_generation_config.temperature);
 
         if let Some(prompt_message) = prompt_message {
-            request.system = prompt_message.message_text;
+            if let LLMMessageContent::Text(text) = &prompt_message.content {
+                request.system = text.clone();
+            }
         }
 
         request.model = text_generation_config.model_id.clone();
@@ -172,15 +176,8 @@ impl ControllerTrait for Controller {
                 ContentBlock::Text { text } => {
                     text_parts.push(text);
                 }
-                ContentBlock::Image {
-                    source,
-                    media_type,
-                    data: _,
-                } => {
-                    text_parts.push(format!(
-                        "The model responded with an image of type {}: {}",
-                        media_type, source
-                    ));
+                ContentBlock::Image { .. } => {
+                    text_parts.push("The model responded with an image".to_string());
                 }
             }
         }
@@ -213,6 +210,15 @@ impl ControllerTrait for Controller {
         Err(anyhow::anyhow!("Image generation not supported"))
     }
 
+    async fn create_image_edit(
+        &self,
+        _prompt: &str,
+        _images: Vec<ImageSource>,
+        _params: ImageEditParams,
+    ) -> anyhow::Result<ImageEditResult> {
+        Err(anyhow::anyhow!("Image editing is not supported"))
+    }
+
     async fn text_to_speech(
         &self,
         _input: &str,

src/agent/provider/anthropic/utils.rs

@@ -1,6 +1,10 @@
-use anthropic::types::{ContentBlock, Message, MessagesRequest, MessagesRequestBuilder, Role};
+use anthropic::types::{
+    ContentBlock, ImageSource, Message, MessagesRequest, MessagesRequestBuilder, Role,
+};
 
-use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage};
+use crate::conversation::llm::{
+    Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent,
+};
 
 pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) -> MessagesRequest {
     let mut messages = vec![];
@@ -14,9 +18,17 @@ pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) ->
             }
         };
 
-        let content = vec![ContentBlock::Text {
-            text: message.message_text,
-        }];
+        let content = match &message.content {
+            LLMMessageContent::Text(text) => vec![ContentBlock::Text { text: text.clone() }],
+            LLMMessageContent::Image(image_details) => {
+                vec![ContentBlock::Image {
+                    source: ImageSource::Base64 {
+                        media_type: image_details.mime.to_string(),
+                        data: crate::utils::base64::base64_encode(&image_details.data),
+                    },
+                }]
+            }
+        };
 
         let message = Message { role, content };
 

src/agent/provider/controller.rs

@@ -1,10 +1,10 @@
 use crate::{agent::AgentPurpose, conversation::llm::Conversation};
 
 use super::{
-    ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
+    ImageEditParams, ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
     entity::{
-        ImageGenerationResult, PingResult, TextGenerationParams, TextGenerationResult,
-        TextToSpeechParams, TextToSpeechResult,
+        ImageEditResult, ImageGenerationResult, ImageSource, PingResult, TextGenerationParams,
+        TextGenerationResult, TextToSpeechParams, TextToSpeechResult,
     },
 };
 
@@ -42,6 +42,13 @@ pub trait ControllerTrait {
         params: ImageGenerationParams,
     ) -> impl std::future::Future<Output = anyhow::Result<ImageGenerationResult>> + Send;
 
+    fn create_image_edit(
+        &self,
+        prompt: &str,
+        images: Vec<ImageSource>,
+        params: ImageEditParams,
+    ) -> impl std::future::Future<Output = anyhow::Result<ImageEditResult>> + Send;
+
     fn text_to_speech(
         &self,
         text: &str,
@@ -166,6 +173,25 @@ impl ControllerTrait for ControllerType {
         }
     }
 
+    async fn create_image_edit(
+        &self,
+        prompt: &str,
+        images: Vec<ImageSource>,
+        params: ImageEditParams,
+    ) -> anyhow::Result<ImageEditResult> {
+        match &self {
+            ControllerType::OpenAI(controller) => {
+                controller.create_image_edit(prompt, images, params).await
+            }
+            ControllerType::OpenAICompat(controller) => {
+                controller.create_image_edit(prompt, images, params).await
+            }
+            ControllerType::Anthropic(controller) => {
+                controller.create_image_edit(prompt, images, params).await
+            }
+        }
+    }
+
     async fn text_to_speech(
         &self,
         text: &str,

src/agent/provider/entity/agent_provider.rs

@@ -68,6 +68,7 @@ impl AgentProvider {
                 sign_up_url: Some("https://console.anthropic.com/"),
                 models_list_url: Some("https://docs.anthropic.com/en/docs/about-claude/models"),
                 supported_purposes: vec![AgentPurpose::TextGeneration],
+                text_generation_supports_vision: true,
             },
             Self::Groq => AgentProviderInfo {
                 id: Self::Groq.to_static_str(),
@@ -78,6 +79,7 @@ impl AgentProvider {
                 sign_up_url: Some("https://console.groq.com/login"),
                 models_list_url: Some("https://console.groq.com/docs/models"),
                 supported_purposes: vec![AgentPurpose::TextGeneration, AgentPurpose::SpeechToText],
+                text_generation_supports_vision: false,
             },
             Self::LocalAI => AgentProviderInfo {
                 id: Self::LocalAI.to_static_str(),
@@ -92,6 +94,7 @@ impl AgentProvider {
                     AgentPurpose::TextToSpeech,
                     AgentPurpose::SpeechToText,
                 ],
+                text_generation_supports_vision: false,
             },
             Self::Mistral => AgentProviderInfo {
                 id: Self::Mistral.to_static_str(),
@@ -102,6 +105,7 @@ impl AgentProvider {
                 sign_up_url: Some("https://auth.mistral.ai/ui/registration"),
                 models_list_url: Some("https://docs.mistral.ai/getting-started/models/"),
                 supported_purposes: vec![AgentPurpose::TextGeneration],
+                text_generation_supports_vision: false,
             },
             Self::Ollama => AgentProviderInfo {
                 id: Self::Ollama.to_static_str(),
@@ -112,6 +116,7 @@ impl AgentProvider {
                 sign_up_url: None,
                 models_list_url: Some("https://ollama.com/library"),
                 supported_purposes: vec![AgentPurpose::TextGeneration],
+                text_generation_supports_vision: false,
             },
             Self::OpenAI => AgentProviderInfo {
                 id: Self::OpenAI.to_static_str(),
@@ -127,6 +132,7 @@ impl AgentProvider {
                     AgentPurpose::TextToSpeech,
                     AgentPurpose::SpeechToText,
                 ],
+                text_generation_supports_vision: true,
             },
             Self::OpenAICompat => AgentProviderInfo {
                 id: Self::OpenAICompat.to_static_str(),
@@ -142,6 +148,7 @@ impl AgentProvider {
                     AgentPurpose::TextToSpeech,
                     AgentPurpose::SpeechToText,
                 ],
+                text_generation_supports_vision: false,
             },
             Self::OpenRouter => AgentProviderInfo {
                 id: Self::OpenRouter.to_static_str(),
@@ -152,6 +159,7 @@ impl AgentProvider {
                 sign_up_url: Some("https://openrouter.ai/"),
                 models_list_url: Some("https://openrouter.ai/models"),
                 supported_purposes: vec![AgentPurpose::TextGeneration],
+                text_generation_supports_vision: false,
             },
             Self::TogetherAI => AgentProviderInfo {
                 id: Self::TogetherAI.to_static_str(),
@@ -162,6 +170,7 @@ impl AgentProvider {
                 sign_up_url: Some("https://api.together.ai/signup"),
                 models_list_url: Some("https://api.together.xyz/models"),
                 supported_purposes: vec![AgentPurpose::TextGeneration],
+                text_generation_supports_vision: false,
             },
         }
     }
@@ -182,4 +191,5 @@ pub struct AgentProviderInfo {
     pub sign_up_url: Option<&'static str>,
     pub models_list_url: Option<&'static str>,
     pub supported_purposes: Vec<AgentPurpose>,
+    pub text_generation_supports_vision: bool,
 }

src/agent/provider/entity/image.rs

@@ -1,3 +1,5 @@
+use mxlink::mime;
+
 #[derive(Default)]
 pub struct ImageGenerationParams {
     pub size_override: Option<String>,
@@ -26,6 +28,40 @@ impl ImageGenerationParams {
 
 pub struct ImageGenerationResult {
     pub bytes: Vec<u8>,
-    pub mime_type: mxlink::mime::Mime,
+    pub mime_type: mime::Mime,
     pub revised_prompt: Option<String>,
 }
+
+#[derive(Default)]
+pub struct ImageEditParams {}
+
+pub struct ImageEditResult {
+    pub bytes: Vec<u8>,
+    pub mime_type: mime::Mime,
+}
+
+pub struct ImageSource {
+    pub filename: String,
+    pub bytes: Vec<u8>,
+    pub mime_type: mime::Mime,
+}
+
+impl ImageSource {
+    pub fn new(filename: String, bytes: Vec<u8>, mime_type: mime::Mime) -> Self {
+        Self {
+            filename,
+            bytes,
+            mime_type,
+        }
+    }
+}
+
+impl From<ImageSource> for async_openai::types::ImageInput {
+    fn from(value: ImageSource) -> Self {
+        async_openai::types::ImageInput::from_vec_u8(
+            value.filename,
+            value.bytes,
+            value.mime_type.to_string(),
+        )
+    }
+}

src/agent/provider/entity/mod.rs

@@ -1,12 +1,14 @@
 mod agent_provider;
-mod image_generation;
+mod image;
 mod ping;
 mod speech_to_text;
 mod text_generation;
 mod text_to_speech;
 
 pub use agent_provider::{AgentProvider, AgentProviderInfo};
-pub use image_generation::{ImageGenerationParams, ImageGenerationResult};
+pub use image::{
+    ImageEditParams, ImageEditResult, ImageGenerationParams, ImageGenerationResult, ImageSource,
+};
 pub use ping::PingResult;
 pub use speech_to_text::{SpeechToTextParams, SpeechToTextResult};
 pub use text_generation::{

src/agent/provider/mod.rs

@@ -20,6 +20,7 @@ pub use controller::{ControllerTrait, ControllerType};
 pub use config::ConfigTrait;
 
 pub use entity::{
-    AgentProvider, AgentProviderInfo, ImageGenerationParams, PingResult, SpeechToTextParams,
-    SpeechToTextResult, TextGenerationParams, TextGenerationPromptVariables, TextToSpeechParams,
+    AgentProvider, AgentProviderInfo, ImageEditParams, ImageGenerationParams, ImageSource,
+    PingResult, SpeechToTextParams, SpeechToTextResult, TextGenerationParams,
+    TextGenerationPromptVariables, TextToSpeechParams,
 };

src/agent/provider/openai/config.rs

@@ -1,5 +1,6 @@
 use serde::{Deserialize, Serialize};
 
+use super::OPENAI_IMAGE_MODEL_GPT_IMAGE_1;
 use crate::agent::{default_prompt, provider::ConfigTrait};
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -71,15 +72,15 @@ impl Default for TextGenerationConfig {
             model_id: default_text_model_id(),
             prompt: Some(default_prompt().to_owned()),
             temperature: super::super::default_temperature(),
-            max_response_tokens: Some(16_384),
-            max_completion_tokens: None,
-            max_context_tokens: 128_000,
+            max_response_tokens: None,
+            max_completion_tokens: Some(128_000),
+            max_context_tokens: 400_000,
         }
     }
 }
 
 fn default_text_model_id() -> String {
-    "gpt-4o".to_owned()
+    "gpt-5.1".to_owned()
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -149,19 +150,19 @@ pub struct ImageGenerationConfig {
     pub model_id: String,
 
     #[serde(default = "default_image_style")]
-    pub style: async_openai::types::ImageStyle,
+    pub style: Option<async_openai::types::ImageStyle>,
 
     #[serde(default = "default_image_size")]
-    pub size: async_openai::types::ImageSize,
+    pub size: Option<async_openai::types::ImageSize>,
 
     #[serde(default = "default_image_quality")]
-    pub quality: async_openai::types::ImageQuality,
+    pub quality: Option<async_openai::types::ImageQuality>,
 }
 
 impl Default for ImageGenerationConfig {
     fn default() -> Self {
         Self {
-            model_id: "dall-e-3".to_owned(),
+            model_id: OPENAI_IMAGE_MODEL_GPT_IMAGE_1.to_owned(),
             style: default_image_style(),
             size: default_image_size(),
             quality: default_image_quality(),
@@ -181,14 +182,14 @@ impl ImageGenerationConfig {
     }
 }
 
-fn default_image_style() -> async_openai::types::ImageStyle {
-    async_openai::types::ImageStyle::Vivid
+fn default_image_style() -> Option<async_openai::types::ImageStyle> {
+    None
 }
 
-fn default_image_size() -> async_openai::types::ImageSize {
-    async_openai::types::ImageSize::S1024x1024
+fn default_image_size() -> Option<async_openai::types::ImageSize> {
+    None
 }
 
-fn default_image_quality() -> async_openai::types::ImageQuality {
-    async_openai::types::ImageQuality::Standard
+fn default_image_quality() -> Option<async_openai::types::ImageQuality> {
+    None
 }

src/agent/provider/openai/controller.rs

@@ -4,38 +4,42 @@ use async_openai::{
     Client as OpenAIClient,
     config::OpenAIConfig,
     types::{
-        ChatCompletionRequestMessage, CreateChatCompletionRequestArgs, CreateImageRequestArgs,
-        CreateSpeechRequestArgs, CreateTranscriptionRequestArgs,
+        ChatCompletionRequestMessage, CreateChatCompletionRequestArgs, CreateImageEditRequestArgs,
+        CreateImageRequestArgs, CreateSpeechRequestArgs, CreateTranscriptionRequestArgs,
+        DallE2ImageSize, Image, ImageModel, ImageResponseFormat,
     },
 };
 
 use super::super::ControllerTrait;
+use crate::{
+    agent::provider::{
+        ImageEditParams, ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
+        entity::{TextGenerationParams, TextGenerationResult},
+    },
+    conversation::llm::{
+        Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
+        MessageContent as LLMMessageContent, shorten_messages_list_to_context_size,
+    },
+    utils::base64::base64_decode,
+};
 use crate::{
     agent::{
         AgentPurpose,
         provider::{
-            entity::{ImageGenerationResult, PingResult, TextToSpeechParams, TextToSpeechResult},
+            entity::{
+                ImageEditResult, ImageGenerationResult, ImageSource, PingResult,
+                TextToSpeechParams, TextToSpeechResult,
+            },
             openai::utils::convert_string_to_enum,
         },
     },
     strings,
 };
-use crate::{
-    agent::{
-        provider::{
-            ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
-            entity::{TextGenerationParams, TextGenerationResult},
-        },
-        utils::base64_decode,
-    },
-    conversation::llm::{
-        Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
-        shorten_messages_list_to_context_size,
-    },
-};
 
 use super::config::Config;
 
+use super::OPENAI_IMAGE_MODEL_GPT_IMAGE_1;
+
 #[derive(Debug, Clone)]
 pub struct Controller {
     config: Config,
@@ -62,7 +66,7 @@ impl ControllerTrait for Controller {
 
         let messages = vec![LLMMessage {
             author: LLMAuthor::User,
-            message_text: "Hello!".to_string(),
+            content: LLMMessageContent::Text("Hello!".to_string()),
             timestamp: chrono::Utc::now(),
         }];
 
@@ -99,7 +103,7 @@ impl ControllerTrait for Controller {
         } else {
             Some(LLMMessage {
                 author: LLMAuthor::Prompt,
-                message_text: prompt_text,
+                content: LLMMessageContent::Text(prompt_text),
                 timestamp: chrono::Utc::now(),
             })
         };
@@ -205,12 +209,11 @@ impl ControllerTrait for Controller {
 
         let request = CreateTranscriptionRequestArgs::default()
             .model(&speech_to_text_config.model_id)
-            .file(async_openai::types::AudioInput {
-                source: async_openai::types::InputSource::VecU8 {
-                    filename,
-                    vec: media,
-                },
-            })
+            .file(async_openai::types::AudioInput::from_vec_u8(
+                filename,
+                media,
+                mime_type.to_string(),
+            ))
             .language(language.clone())
             .build()?;
 
@@ -265,12 +268,15 @@ impl ControllerTrait for Controller {
         let quality = if params.cheaper_quality_switching_allowed {
             // Switch to a cheaper quality
             match &image_generation_config.quality {
-                async_openai::types::ImageQuality::Standard => {
-                    async_openai::types::ImageQuality::Standard
-                }
-                async_openai::types::ImageQuality::HD => {
-                    async_openai::types::ImageQuality::Standard
-                }
+                Some(quality) => match quality {
+                    async_openai::types::ImageQuality::Standard => {
+                        Some(async_openai::types::ImageQuality::Standard)
+                    }
+                    async_openai::types::ImageQuality::HD => {
+                        Some(async_openai::types::ImageQuality::Standard)
+                    }
+                },
+                None => None,
             }
         } else {
             image_generation_config.quality.clone()
@@ -278,20 +284,41 @@ impl ControllerTrait for Controller {
 
         let size = params
             .size_override
-            .map(|s| {
-                convert_string_to_enum::<async_openai::types::ImageSize>(&s)
-                    .unwrap_or(image_generation_config.size)
-            })
-            .unwrap_or(image_generation_config.size);
+            .map(|s| convert_string_to_enum::<async_openai::types::ImageSize>(&s).unwrap())
+            .or(image_generation_config.size);
 
-        let request = CreateImageRequestArgs::default()
-            .model(model)
-            .prompt(prompt.to_owned())
-            .response_format(async_openai::types::ImageResponseFormat::B64Json)
-            .size(size)
-            .style(image_generation_config.style.clone())
-            .quality(quality)
-            .build()?;
+        let response_format = match model.clone() {
+            ImageModel::DallE2 => Some(ImageResponseFormat::B64Json),
+            ImageModel::DallE3 => Some(ImageResponseFormat::B64Json),
+            ImageModel::Other(model_str) => match model_str.as_str() {
+                // gpt-image-1 only outputs base64 and we don't need to specify the response format.
+                // In fact, specifying the response format results in an error.
+                OPENAI_IMAGE_MODEL_GPT_IMAGE_1 => None,
+                _ => Some(ImageResponseFormat::B64Json),
+            },
+        };
+
+        let mut request_builder = CreateImageRequestArgs::default();
+
+        request_builder.model(model).prompt(prompt.to_owned());
+
+        if let Some(response_format) = response_format {
+            request_builder.response_format(response_format);
+        }
+
+        if let Some(style) = &image_generation_config.style {
+            request_builder.style(style.clone());
+        }
+
+        if let Some(quality) = quality {
+            request_builder.quality(quality.clone());
+        }
+
+        if let Some(size) = size {
+            request_builder.size(size);
+        }
+
+        let request = request_builder.build()?;
 
         tracing::trace!(
             ?prompt,
@@ -329,6 +356,104 @@ impl ControllerTrait for Controller {
         ))
     }
 
+    async fn create_image_edit(
+        &self,
+        prompt: &str,
+        images: Vec<ImageSource>,
+        _params: ImageEditParams,
+    ) -> anyhow::Result<ImageEditResult> {
+        let Some(image_generation_config) = &self.config.image_generation else {
+            return Err(anyhow::anyhow!(
+                strings::agent::no_configuration_for_purpose_so_cannot_be_used(
+                    &AgentPurpose::ImageGeneration
+                ),
+            ));
+        };
+
+        if images.is_empty() {
+            return Err(anyhow::anyhow!("No image sources provided"));
+        }
+
+        let mut image_inputs = Vec::new();
+        for image in images {
+            image_inputs.push(image.into());
+        }
+
+        let dalle2_size = match image_generation_config.size {
+            Some(async_openai::types::ImageSize::S256x256) => Some(DallE2ImageSize::S256x256),
+            Some(async_openai::types::ImageSize::S512x512) => Some(DallE2ImageSize::S512x512),
+            Some(async_openai::types::ImageSize::S1024x1024) => Some(DallE2ImageSize::S1024x1024),
+            _ => None,
+        };
+
+        let model = image_generation_config
+            .model_id_as_openai_image_model()
+            .map_err(|err| anyhow::anyhow!(err))?;
+
+        let response_format = match model.clone() {
+            async_openai::types::ImageModel::DallE2 => {
+                Some(async_openai::types::ImageResponseFormat::B64Json)
+            }
+            async_openai::types::ImageModel::DallE3 => {
+                Some(async_openai::types::ImageResponseFormat::B64Json)
+            }
+            async_openai::types::ImageModel::Other(model_str) => match model_str.as_str() {
+                OPENAI_IMAGE_MODEL_GPT_IMAGE_1 => None,
+                _ => Some(async_openai::types::ImageResponseFormat::B64Json),
+            },
+        };
+
+        let mut request_builder = CreateImageEditRequestArgs::default();
+
+        request_builder
+            .image(image_inputs)
+            .prompt(prompt.to_owned())
+            .model(model);
+
+        if let Some(size) = dalle2_size {
+            request_builder.size(size);
+        }
+
+        if let Some(response_format) = response_format {
+            request_builder.response_format(response_format);
+        }
+
+        let request = request_builder
+            .build()
+            .map_err(|e| anyhow::anyhow!("Failed to build CreateImageEditRequest: {}", e))?;
+
+        tracing::trace!(
+            model = format!("{:?}", request.model),
+            size = format!("{:?}", request.size),
+            response_format = format!("{:?}", request.response_format),
+            "Sending OpenAI image edit API request"
+        );
+
+        let response = self.client.images().create_edit(request).await?;
+
+        if let Some(image_data) = response.data.into_iter().next() {
+            match image_data.deref() {
+                Image::B64Json { b64_json, .. } => {
+                    let bytes = base64_decode(b64_json)?;
+                    return Ok(ImageEditResult {
+                        bytes,
+                        mime_type: mxlink::mime::IMAGE_PNG,
+                    });
+                }
+                Image::Url { url, .. } => {
+                    tracing::warn!(?url, "Received URL instead of B64Json for image edit");
+                    return Err(anyhow::anyhow!(
+                        "Unexpected image type (URL) when B64Json was requested"
+                    ));
+                }
+            }
+        }
+
+        Err(anyhow::anyhow!(
+            "The OpenAI image edit API returned no images"
+        ))
+    }
+
     async fn text_to_speech(
         &self,
         input: &str,

src/agent/provider/openai/mod.rs

@@ -16,6 +16,8 @@ use super::super::AgentInstantiationResult;
 use super::ConfigTrait;
 use super::controller::ControllerType;
 
+pub const OPENAI_IMAGE_MODEL_GPT_IMAGE_1: &str = "gpt-image-1";
+
 pub fn create_controller_from_yaml_value_config(
     agent_id: &str,
     config: serde_yaml::Value,

src/agent/provider/openai/utils.rs

@@ -1,9 +1,14 @@
 use async_openai::types::{
     ChatCompletionRequestAssistantMessageArgs, ChatCompletionRequestMessage,
-    ChatCompletionRequestSystemMessageArgs, ChatCompletionRequestUserMessageArgs,
+    ChatCompletionRequestMessageContentPartImage, ChatCompletionRequestSystemMessageArgs,
+    ChatCompletionRequestUserMessageArgs, ChatCompletionRequestUserMessageContent,
+    ChatCompletionRequestUserMessageContentPart, ImageUrlArgs,
 };
 
-use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage};
+use crate::conversation::llm::{
+    Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent,
+};
+use crate::utils::base64::base64_encode;
 
 pub fn convert_llm_messages_to_openai_messages(
     conversation_messages: Vec<LLMMessage>,
@@ -12,29 +17,71 @@ pub fn convert_llm_messages_to_openai_messages(
         Vec::with_capacity(conversation_messages.len());
 
     for message in conversation_messages {
-        openai_conversation_messages.push(convert_llm_message_to_openai_message(message));
+        let openai_message = convert_llm_message_to_openai_message(message);
+        if let Some(openai_message) = openai_message {
+            openai_conversation_messages.push(openai_message);
+        }
     }
 
     openai_conversation_messages
 }
 
-fn convert_llm_message_to_openai_message(llm_message: LLMMessage) -> ChatCompletionRequestMessage {
-    match llm_message.author {
-        LLMAuthor::Prompt => ChatCompletionRequestSystemMessageArgs::default()
-            .content(llm_message.message_text)
-            .build()
-            .expect("Failed building OpenAI system message")
-            .into(),
-        LLMAuthor::Assistant => ChatCompletionRequestAssistantMessageArgs::default()
-            .content(llm_message.message_text)
-            .build()
-            .expect("Failed building OpenAI assistant message")
-            .into(),
-        LLMAuthor::User => ChatCompletionRequestUserMessageArgs::default()
-            .content(llm_message.message_text)
-            .build()
-            .expect("Failed building OpenAI user message")
-            .into(),
+fn convert_llm_message_to_openai_message(
+    llm_message: LLMMessage,
+) -> Option<ChatCompletionRequestMessage> {
+    match &llm_message.content {
+        LLMMessageContent::Text(text) => Some(match llm_message.author {
+            LLMAuthor::Prompt => ChatCompletionRequestSystemMessageArgs::default()
+                .content(text.clone())
+                .build()
+                .expect("Failed building OpenAI system message")
+                .into(),
+            LLMAuthor::Assistant => ChatCompletionRequestAssistantMessageArgs::default()
+                .content(text.clone())
+                .build()
+                .expect("Failed building OpenAI assistant message")
+                .into(),
+            LLMAuthor::User => ChatCompletionRequestUserMessageArgs::default()
+                .content(text.clone())
+                .build()
+                .expect("Failed building OpenAI user message")
+                .into(),
+        }),
+        LLMMessageContent::Image(image_details) => {
+            let image_url = format!(
+                "data:{};base64,{}",
+                image_details.mime,
+                base64_encode(&image_details.data)
+            );
+
+            let part = ChatCompletionRequestUserMessageContentPart::ImageUrl(
+                ChatCompletionRequestMessageContentPartImage {
+                    image_url: ImageUrlArgs::default()
+                        .url(image_url)
+                        .build()
+                        .expect("Failed building OpenAI image url"),
+                },
+            );
+
+            let message_content = ChatCompletionRequestUserMessageContent::Array(vec![part]);
+
+            match llm_message.author {
+                LLMAuthor::User => Some(
+                    ChatCompletionRequestUserMessageArgs::default()
+                        .content(message_content)
+                        .build()
+                        .expect("Failed building OpenAI user message")
+                        .into(),
+                ),
+                _ => {
+                    tracing::warn!(
+                        "OpenAI API does not support image content for messages authored by {:?}. This message part will be skipped.",
+                        llm_message.author
+                    );
+                    None
+                }
+            }
+        }
     }
 }
 

src/agent/provider/openai_compat/config.rs

@@ -224,21 +224,27 @@ impl TryInto<OpenAIImageGenerationConfig> for ImageGenerationConfig {
 
     fn try_into(self) -> Result<OpenAIImageGenerationConfig, Self::Error> {
         let size = if let Some(size) = &self.size {
-            convert_string_to_enum::<async_openai::types::ImageSize>(size)?
+            Some(convert_string_to_enum::<async_openai::types::ImageSize>(
+                size,
+            )?)
         } else {
-            async_openai::types::ImageSize::S1024x1024
+            None
         };
 
         let style = if let Some(style) = &self.style {
-            convert_string_to_enum::<async_openai::types::ImageStyle>(style)?
+            Some(convert_string_to_enum::<async_openai::types::ImageStyle>(
+                style,
+            )?)
         } else {
-            async_openai::types::ImageStyle::Vivid
+            None
         };
 
         let quality = if let Some(quality) = &self.quality {
-            convert_string_to_enum::<async_openai::types::ImageQuality>(quality)?
+            Some(convert_string_to_enum::<async_openai::types::ImageQuality>(
+                quality,
+            )?)
         } else {
-            async_openai::types::ImageQuality::Standard
+            None
         };
 
         Ok(OpenAIImageGenerationConfig {

src/agent/provider/openai_compat/controller.rs

@@ -4,22 +4,24 @@ use etke_openai_api_rust::images::{ImagesApi, ImagesBody};
 use etke_openai_api_rust::{Auth, Message, OpenAI};
 
 use super::super::ControllerTrait;
-use crate::agent::utils::base64_decode;
+use crate::utils::base64::base64_decode;
 use crate::{
     agent::provider::{
-        ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
+        ImageEditParams, ImageGenerationParams, ImageSource, SpeechToTextParams,
+        SpeechToTextResult,
         entity::{TextGenerationParams, TextGenerationResult},
     },
     conversation::llm::{
         Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
-        shorten_messages_list_to_context_size,
+        MessageContent as LLMMessageContent, shorten_messages_list_to_context_size,
     },
 };
 use crate::{
     agent::{
         AgentPurpose,
         provider::entity::{
-            ImageGenerationResult, PingResult, TextToSpeechParams, TextToSpeechResult,
+            ImageEditResult, ImageGenerationResult, PingResult, TextToSpeechParams,
+            TextToSpeechResult,
         },
     },
     strings,
@@ -60,7 +62,7 @@ impl ControllerTrait for Controller {
 
         let messages = vec![LLMMessage {
             author: LLMAuthor::User,
-            message_text: "Hello!".to_string(),
+            content: LLMMessageContent::Text("Hello!".to_string()),
             timestamp: chrono::Utc::now(),
         }];
 
@@ -97,7 +99,7 @@ impl ControllerTrait for Controller {
         } else {
             Some(LLMMessage {
                 author: LLMAuthor::Prompt,
-                message_text: prompt_text,
+                content: LLMMessageContent::Text(prompt_text),
                 timestamp: chrono::Utc::now(),
             })
         };
@@ -366,6 +368,17 @@ impl ControllerTrait for Controller {
         ))
     }
 
+    async fn create_image_edit(
+        &self,
+        _prompt: &str,
+        _images: Vec<ImageSource>,
+        _params: ImageEditParams,
+    ) -> anyhow::Result<ImageEditResult> {
+        Err(anyhow::anyhow!(
+            "The OpenAI image edit API is not supported by the OpenAI-compat provider"
+        ))
+    }
+
     async fn text_to_speech(
         &self,
         input: &str,

src/agent/provider/openai_compat/utils.rs

@@ -2,7 +2,9 @@ use etke_openai_api_rust::{Message, Role};
 
 use crate::agent::provider::openai::Config as OpenAIConfig;
 
-use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage};
+use crate::conversation::llm::{
+    Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent,
+};
 
 pub fn convert_llm_messages_to_openai_messages(
     conversation_messages: Vec<LLMMessage>,
@@ -11,22 +13,33 @@ pub fn convert_llm_messages_to_openai_messages(
         Vec::with_capacity(conversation_messages.len());
 
     for message in conversation_messages {
-        openai_conversation_messages.push(convert_llm_message_to_openai_message(message));
+        let openai_message = convert_llm_message_to_openai_message(message);
+        if let Some(openai_message) = openai_message {
+            openai_conversation_messages.push(openai_message);
+        }
     }
 
     openai_conversation_messages
 }
 
-fn convert_llm_message_to_openai_message(llm_message: LLMMessage) -> Message {
+fn convert_llm_message_to_openai_message(llm_message: LLMMessage) -> Option<Message> {
     let role = match llm_message.author {
         LLMAuthor::Prompt => Role::System,
         LLMAuthor::Assistant => Role::Assistant,
         LLMAuthor::User => Role::User,
     };
 
-    Message {
-        role,
-        content: llm_message.message_text,
+    match &llm_message.content {
+        LLMMessageContent::Text(text) => Some(Message {
+            role,
+            content: text.clone(),
+        }),
+        LLMMessageContent::Image(_image_details) => {
+            tracing::warn!(
+                "The OpenAI-compat provider's library does not support image content. This image message will be skipped."
+            );
+            None
+        }
     }
 }
 

src/agent/utils.rs

@@ -1,5 +1,3 @@
-use base64::{Engine as _, engine::general_purpose::STANDARD};
-
 use crate::{
     agent::{
         AgentInstance, AgentPurpose, ControllerTrait, Manager as AgentManager, PublicIdentifier,
@@ -140,7 +138,3 @@ async fn get_global_agent_id_for_purpose(
         .handler
         .get_by_purpose_with_catch_all_fallback(purpose)
 }
-
-pub(crate) fn base64_decode(base64_string: &str) -> Result<Vec<u8>, base64::DecodeError> {
-    STANDARD.decode(base64_string)
-}

src/bot/implementation.rs

@@ -6,6 +6,7 @@ use mxlink::matrix_sdk::media::{MediaFormat, MediaRequestParameters};
 use mxlink::matrix_sdk::ruma::{
     MilliSecondsSinceUnixEpoch, OwnedUserId, events::room::MediaSource,
 };
+use mxlink::matrix_sdk::ruma::api::client::profile::{AvatarUrl, DisplayName};
 
 use mxlink::{
     InitConfig, LoginConfig, LoginCredentials, LoginEncryption, MatrixLink, PersistenceConfig,
@@ -285,24 +286,27 @@ impl Bot {
     async fn do_prepare_profile(&self) -> anyhow::Result<()> {
         tracing::debug!("Preparing profile..");
 
+        let desired_display_name = self.inner.config.user.name.clone();
+
         let account = self.inner.matrix_link.client().account();
         let media = self.inner.matrix_link.client().media();
 
-        let desired_display_name = self.inner.config.user.name.clone();
-
         let profile = account
             .fetch_user_profile()
             .await
             .map_err(|e| anyhow::anyhow!("Failed fetching profile: {:?}", e))?;
 
-        let should_update_display_name = match &profile.displayname {
+        let current_display_name = profile.get_static::<DisplayName>()?;
+        let current_avatar_url = profile.get_static::<AvatarUrl>()?;
+
+        let should_update_display_name = match &current_display_name {
             Some(displayname) => displayname != &desired_display_name,
             None => true,
         };
 
         if should_update_display_name {
             tracing::info!(
-                ?profile.displayname,
+                ?current_display_name,
                 ?desired_display_name,
                 "Updating display name.."
             );
@@ -312,7 +316,7 @@ impl Bot {
             }
         }
 
-        let should_update_avatar = match &profile.avatar_url {
+        let should_update_avatar = match &current_avatar_url {
             Some(avatar_url) => {
                 let request = MediaRequestParameters {
                     source: MediaSource::Plain(avatar_url.to_owned()),

src/controller/cfg/status.rs

@@ -69,7 +69,7 @@ pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Resu
     );
     message.push_str("\n\n");
 
-    // Image Generation
+    // Image Creation
     message.push_str(
         &generate_image_generation_section(agent_manager, message_context.room_config_context())
             .await,

src/controller/chat_completion/mod.rs

@@ -39,6 +39,8 @@ pub enum ChatCompletionControllerType {
 
     Audio,
 
+    Image,
+
     ThreadMention,
     ReplyMention,
 }
@@ -416,7 +418,8 @@ async fn handle_stage_text_generation(
         ChatCompletionControllerType::TextCommand
         | ChatCompletionControllerType::TextMention
         | ChatCompletionControllerType::TextDirect
-        | ChatCompletionControllerType::Audio => {
+        | ChatCompletionControllerType::Audio
+        | ChatCompletionControllerType::Image => {
             Some(message_context.combined_admin_and_user_regexes())
         }
 
@@ -438,6 +441,7 @@ async fn handle_stage_text_generation(
         // When we're triggered via a reply mention, the context is the whole reply chain upward of the message that triggered us.
         ChatCompletionControllerType::ReplyMention => {
             create_llm_conversation_for_matrix_reply_chain(
+                &matrix_link,
                 &bot.room_event_fetcher().clone(),
                 message_context.room(),
                 message_context.thread_info().last_event_id.clone(),
@@ -449,7 +453,7 @@ async fn handle_stage_text_generation(
         // Everything else is happening in a thread, so the context is the whole thread.
         _ => {
             create_llm_conversation_for_matrix_thread(
-                matrix_link.clone(),
+                &matrix_link,
                 message_context.room(),
                 message_context.thread_info().root_event_id.clone(),
                 &params,

src/controller/controller_type.rs

@@ -23,5 +23,6 @@ pub enum ControllerType {
     ChatCompletion(super::chat_completion::ChatCompletionControllerType),
 
     ImageGeneration(String),
+    ImageEdit(String),
     StickerGeneration(String),
 }

src/controller/determination/mod.rs

@@ -36,6 +36,18 @@ pub fn determine_controller(
                 first_thread_message.is_mentioning_bot,
             )
         }
+        MessagePayload::Image(_image_message_content) => {
+            let prefix_requirement_type = message_context
+                .room_config_context()
+                .text_generation_prefix_requirement_type();
+
+            match prefix_requirement_type {
+                TextGenerationPrefixRequirementType::CommandPrefix => ControllerType::Ignore,
+                TextGenerationPrefixRequirementType::No => {
+                    ControllerType::ChatCompletion(ChatCompletionControllerType::Image)
+                }
+            }
+        }
         MessagePayload::Encrypted(thread_info) => {
             if thread_info.is_thread_root_only() {
                 ControllerType::Error(strings::error::message_is_encrypted().to_owned())
@@ -84,7 +96,7 @@ fn determine_text_controller(
     }
 
     if let Some(prompt) = text.strip_prefix(&format!("{command_prefix} image")) {
-        return ControllerType::ImageGeneration(prompt.trim().to_owned());
+        return super::image::determine_controller(prompt.trim());
     }
 
     if let Some(prompt) = text.strip_prefix(&format!("{command_prefix} sticker")) {

src/controller/determination/tests.rs

@@ -84,9 +84,17 @@ fn determine_text_controller() {
             expected: ControllerType::Config(controller::cfg::ConfigControllerType::Help),
         },
         TestCase {
-            name: "Image generation",
+            name: "Generic image command causes usage help",
             input: "!bai image Draw a cat!",
             is_mentioning_bot: false,
+            room_text_generation_prefix_requirement_type:
+                super::TextGenerationPrefixRequirementType::No,
+            expected: ControllerType::UsageHelp,
+        },
+        TestCase {
+            name: "Image generation",
+            input: "!bai image create Draw a cat!",
+            is_mentioning_bot: false,
             room_text_generation_prefix_requirement_type:
                 super::TextGenerationPrefixRequirementType::No,
             expected: ControllerType::ImageGeneration("Draw a cat!".to_owned()),

src/controller/dispatching.rs

@@ -77,6 +77,10 @@ pub async fn dispatch_controller(
             )
             .await
         }
+        ControllerType::ImageEdit(prompt) => {
+            super::image::edit::handle(bot, bot.matrix_link().clone(), message_context, prompt)
+                .await
+        }
         ControllerType::StickerGeneration(prompt) => {
             super::image::generation::handle_sticker(
                 bot,

src/controller/image/determination/mod.rs

@@ -0,0 +1,16 @@
+use crate::controller::ControllerType;
+mod tests;
+
+pub fn determine_controller(text: &str) -> ControllerType {
+    let text = text.trim();
+
+    if let Some(prompt) = text.strip_prefix("create") {
+        return ControllerType::ImageGeneration(prompt.trim().to_owned());
+    }
+
+    if let Some(prompt) = text.strip_prefix("edit") {
+        return ControllerType::ImageEdit(prompt.trim().to_owned());
+    }
+
+    ControllerType::UsageHelp
+}

src/controller/image/determination/tests.rs

@@ -0,0 +1,38 @@
+#[test]
+fn determine_controller() {
+    struct TestCase {
+        name: &'static str,
+        input: &'static str,
+        expected: super::ControllerType,
+    }
+
+    let test_cases = vec![
+        TestCase {
+            name: "Top-level is usage help",
+            input: "",
+            expected: super::ControllerType::UsageHelp,
+        },
+        TestCase {
+            name: "Top-level with some text is usage help",
+            input: "Some text",
+            expected: super::ControllerType::UsageHelp,
+        },
+        TestCase {
+            name: "Image generation triggered by create prefix",
+            input: "create Some prompt",
+            expected: super::ControllerType::ImageGeneration("Some prompt".to_owned()),
+        },
+        TestCase {
+            name: "Image edit triggered by edit prefix",
+            input: "edit Turn this into an anime-style image",
+            expected: super::ControllerType::ImageEdit(
+                "Turn this into an anime-style image".to_owned(),
+            ),
+        },
+    ];
+
+    for test_case in test_cases {
+        let result = super::determine_controller(test_case.input);
+        assert_eq!(result, test_case.expected, "Test case: {}", test_case.name);
+    }
+}

src/controller/image/edit.rs

@@ -0,0 +1,162 @@
+use mxlink::{MatrixLink, MessageResponseType};
+
+use tracing::Instrument;
+
+use crate::agent::AgentPurpose;
+use crate::agent::ControllerTrait;
+use crate::agent::provider::ImageEditParams;
+use crate::agent::provider::ImageSource;
+use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain;
+use crate::conversation::create_llm_conversation_for_matrix_thread;
+use crate::conversation::matrix::MatrixMessageProcessingParams;
+use crate::strings;
+use crate::utils::mime::get_file_extension;
+use crate::{Bot, entity::MessageContext};
+
+pub async fn handle(
+    bot: &Bot,
+    matrix_link: MatrixLink,
+    message_context: &MessageContext,
+    original_prompt: &str,
+) -> anyhow::Result<()> {
+    let response_type = MessageResponseType::InThread(message_context.thread_info().clone());
+
+    let Some(agent) = get_effective_agent_for_purpose_or_complain(
+        bot,
+        message_context,
+        AgentPurpose::ImageGeneration,
+        response_type.clone(),
+        true,
+    )
+    .await
+    else {
+        return Ok(());
+    };
+
+    if message_context.thread_info().is_thread_root_only() {
+        return send_guide(bot, message_context).await;
+    }
+
+    let _typing_notice_guard = bot.start_typing_notice(message_context.room()).await;
+
+    let params = MatrixMessageProcessingParams::new(
+        bot.user_id().to_owned(),
+        Some(message_context.combined_admin_and_user_regexes()),
+    );
+
+    let conversation = create_llm_conversation_for_matrix_thread(
+        &matrix_link,
+        message_context.room(),
+        message_context.thread_info().root_event_id.clone(),
+        &params,
+    )
+    .await?;
+
+    let prompt = if conversation.messages.len() >= 2 {
+        // Skip the first message, which contains the original prompt (which we already have)
+        let other_messages = conversation.messages.iter().skip(1).cloned().collect();
+
+        super::prompt::build(original_prompt, other_messages)
+    } else {
+        original_prompt.to_owned()
+    };
+
+    let got_go_signal = conversation.messages.iter().any(|message| {
+        if let crate::conversation::llm::MessageContent::Text(text) = &message.content {
+            text.to_lowercase() == "go"
+        } else {
+            false
+        }
+    });
+
+    let image_sources: Vec<ImageSource> = conversation
+        .messages
+        .iter()
+        .filter_map(|message| {
+            if let crate::conversation::llm::MessageContent::Image(image_content) = &message.content
+            {
+                Some(image_content.clone().into())
+            } else {
+                None
+            }
+        })
+        .collect();
+
+    if !got_go_signal || image_sources.is_empty() {
+        // We don't send the guide again here to avoid being annoying.
+        return Ok(());
+    }
+
+    let span = tracing::debug_span!("image_edit", agent_id = agent.identifier().as_string());
+
+    let result = agent
+        .controller()
+        .create_image_edit(&prompt, image_sources, ImageEditParams::default())
+        .instrument(span)
+        .await;
+
+    let response = match result {
+        Ok(response) => response,
+        Err(err) => {
+            tracing::warn!(
+                "Error in room {} while trying to generate image edit via agent {}: {:?}",
+                message_context.room_id(),
+                agent.identifier(),
+                err,
+            );
+
+            bot.messaging()
+                .send_error_markdown_no_fail(
+                    message_context.room(),
+                    &strings::agent::error_while_serving_purpose(
+                        agent.identifier(),
+                        &AgentPurpose::ImageGeneration,
+                        &err,
+                    ),
+                    response_type,
+                )
+                .await;
+
+            return Ok(());
+        }
+    };
+
+    let attachment_body_text = format!(
+        "generated-image-edit.{}",
+        get_file_extension(&response.mime_type)
+    );
+
+    let mut event_content = matrix_link
+        .media()
+        .upload_and_prepare_event_content(
+            message_context.room(),
+            &response.mime_type,
+            response.bytes,
+            &attachment_body_text,
+        )
+        .await
+        .map_err(|e| anyhow::anyhow!("Failed to upload and prepare event: {}", e))?;
+
+    matrix_link
+        .messaging()
+        .send_event(
+            message_context.room(),
+            &mut event_content,
+            response_type.clone(),
+        )
+        .await?;
+
+    Ok(())
+}
+
+async fn send_guide(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> {
+    bot.messaging()
+        .send_text_markdown_no_fail(
+            message_context.room(),
+            strings::image_edit::guide_how_to_proceed(),
+            MessageResponseType::InThread(message_context.thread_info().clone()),
+        )
+        .await;
+
+    Ok(())
+}

src/controller/image/generation.rs

@@ -6,10 +6,10 @@ use crate::agent::AgentPurpose;
 use crate::agent::ControllerTrait;
 use crate::agent::provider::ImageGenerationParams;
 use crate::controller::utils::agent::get_effective_agent_for_purpose_or_complain;
-use crate::controller::utils::mime::get_file_extension;
 use crate::conversation::create_llm_conversation_for_matrix_thread;
 use crate::conversation::matrix::MatrixMessageProcessingParams;
 use crate::strings;
+use crate::utils::mime::get_file_extension;
 use crate::{Bot, entity::MessageContext};
 
 // We may make this configurable (per room, etc.) in the future, but for now it's hardcoded.
@@ -43,7 +43,7 @@ pub async fn handle_image(
     );
 
     let conversation = create_llm_conversation_for_matrix_thread(
-        matrix_link.clone(),
+        &matrix_link,
         message_context.room(),
         message_context.thread_info().root_event_id.clone(),
         &params,
@@ -64,11 +64,37 @@ pub async fn handle_image(
         agent_id = agent.identifier().as_string()
     );
 
-    let response = agent
+    let result = agent
         .controller()
         .generate_image(&prompt, ImageGenerationParams::default())
         .instrument(span)
-        .await?;
+        .await;
+
+    let response = match result {
+        Ok(response) => response,
+        Err(err) => {
+            tracing::warn!(
+                "Error in room {} while trying to generate image via agent {}: {:?}",
+                message_context.room_id(),
+                agent.identifier(),
+                err,
+            );
+
+            bot.messaging()
+                .send_error_markdown_no_fail(
+                    message_context.room(),
+                    &strings::agent::error_while_serving_purpose(
+                        agent.identifier(),
+                        &AgentPurpose::ImageGeneration,
+                        &err,
+                    ),
+                    response_type,
+                )
+                .await;
+
+            return Ok(());
+        }
+    };
 
     let actual_prompt = response.revised_prompt.as_deref().unwrap_or(&prompt);
 
@@ -155,11 +181,37 @@ pub async fn handle_sticker(
         .with_cheaper_model_switching_allowed(true)
         .with_cheaper_quality_switching_allowed(true);
 
-    let response = agent
+    let result = agent
         .controller()
         .generate_image(original_prompt, params)
         .instrument(span)
-        .await?;
+        .await;
+
+    let response = match result {
+        Ok(response) => response,
+        Err(err) => {
+            tracing::warn!(
+                "Error in room {} while trying to generate sticker via agent {}: {:?}",
+                message_context.room_id(),
+                agent.identifier(),
+                err,
+            );
+
+            bot.messaging()
+                .send_error_markdown_no_fail(
+                    message_context.room(),
+                    &strings::agent::error_while_serving_purpose(
+                        agent.identifier(),
+                        &AgentPurpose::ImageGeneration,
+                        &err,
+                    ),
+                    response_type,
+                )
+                .await;
+
+            return Ok(());
+        }
+    };
 
     let attachment_body_text = format!(
         "generated-sticker.{}",

src/controller/image/mod.rs

@@ -1,2 +1,6 @@
+mod determination;
+pub mod edit;
 pub mod generation;
 mod prompt;
+
+pub use determination::determine_controller;

src/controller/image/prompt.rs

@@ -1,11 +1,11 @@
-use crate::conversation::llm::{Author, Message};
+use crate::conversation::llm::{Author, Message, MessageContent};
 
 /// Builds a prompt from the original prompt and other messages in the conversation.
 ///
 /// Only messages authored by the user are considered.
 ///
-/// Messages that say "Again" (regardless of casing) are ignored. They are considered special messages
-/// which trigger re-generation, but do not need to be included in the prompt criteria.
+/// Messages that say "Again" or "Go" (regardless of casing) are ignored. They are considered special messages
+/// which trigger re-generation and "start" respectively, and do not need to be included in the prompt criteria.
 pub fn build(original_prompt: &str, other_messages: Vec<Message>) -> String {
     let mut prompt = original_prompt.to_owned();
 
@@ -14,7 +14,11 @@ pub fn build(original_prompt: &str, other_messages: Vec<Message>) -> String {
         .into_iter()
         .filter(|message| {
             if let Author::User = message.author {
-                message.message_text.to_lowercase() != "again"
+                if let MessageContent::Text(text) = &message.content {
+                    text.to_lowercase() != "again" && text.to_lowercase() != "go"
+                } else {
+                    false
+                }
             } else {
                 false
             }
@@ -24,9 +28,9 @@ pub fn build(original_prompt: &str, other_messages: Vec<Message>) -> String {
     if !other_messages.is_empty() {
         prompt.push_str("\nOther criteria:");
         for message in other_messages {
-            prompt.push_str(
-                format!("\n- {}", message.message_text.replace("\n", ". ").as_str()).as_str(),
-            );
+            if let MessageContent::Text(text) = &message.content {
+                prompt.push_str(format!("\n- {}", text.replace("\n", ". ").as_str()).as_str());
+            }
         }
     }
 
@@ -36,7 +40,7 @@ pub fn build(original_prompt: &str, other_messages: Vec<Message>) -> String {
 #[cfg(test)]
 mod tests {
     use super::build;
-    use super::{Author, Message};
+    use super::{Author, Message, MessageContent};
 
     struct TestCase {
         original_prompt: &'static str,
@@ -60,7 +64,7 @@ mod tests {
                 original_prompt: "Generate a picture of a dog",
                 messages: vec![Message {
                     author: Author::User,
-                    message_text: "Must be blue".to_owned(),
+                    content: MessageContent::Text("Must be blue".to_owned()),
                     timestamp,
                 }],
                 expected_prompt: "Generate a picture of a dog\nOther criteria:\n- Must be blue",
@@ -71,17 +75,19 @@ mod tests {
                 messages: vec![
                     Message {
                         author: Author::User,
-                        message_text: "Must be blue".to_owned(),
+                        content: MessageContent::Text("Must be blue".to_owned()),
                         timestamp,
                     },
                     Message {
                         author: Author::Assistant,
-                        message_text: "Whatever".to_owned(),
+                        content: MessageContent::Text("Whatever".to_owned()),
                         timestamp,
                     },
                     Message {
                         author: Author::User,
-                        message_text: "Must be 3-legged.\nMust be flying.".to_owned(),
+                        content: MessageContent::Text(
+                            "Must be 3-legged.\nMust be flying.".to_owned(),
+                        ),
                         timestamp,
                     },
                 ],
@@ -93,22 +99,22 @@ mod tests {
                 messages: vec![
                     Message {
                         author: Author::User,
-                        message_text: "Must be blue".to_owned(),
+                        content: MessageContent::Text("Must be blue".to_owned()),
                         timestamp,
                     },
                     Message {
                         author: Author::Assistant,
-                        message_text: "Whatever".to_owned(),
+                        content: MessageContent::Text("Whatever".to_owned()),
                         timestamp,
                     },
                     Message {
                         author: Author::User,
-                        message_text: "Again".to_owned(),
+                        content: MessageContent::Text("Again".to_owned()),
                         timestamp,
                     },
                     Message {
                         author: Author::User,
-                        message_text: "again".to_owned(),
+                        content: MessageContent::Text("again".to_owned()),
                         timestamp,
                     },
                 ],

src/controller/utils/mime.rs

@@ -1,17 +0,0 @@
-use mxlink::mime;
-
-pub fn get_file_extension(mime_type: &mime::Mime) -> String {
-    match (mime_type.type_(), mime_type.subtype()) {
-        (mime::AUDIO, mime::BASIC) => "au",
-        (mime::AUDIO, mime::MPEG) => "mp3",
-        (mime::AUDIO, mime::MP4) => "m4a",
-        (mime::AUDIO, mime::OGG) => "ogg",
-        (mime::IMAGE, mime::BMP) => "bmp",
-        (mime::IMAGE, mime::GIF) => "gif",
-        (mime::IMAGE, mime::JPEG) => "jpg",
-        (mime::IMAGE, mime::PNG) => "png",
-        (mime::IMAGE, mime::SVG) => "svg",
-        _ => "bin",
-    }
-    .to_string()
-}

src/controller/utils/mod.rs

@@ -6,7 +6,6 @@ use crate::{
 };
 
 pub mod agent;
-pub(super) mod mime;
 pub mod text_to_speech;
 
 pub async fn get_text_body_or_complain<'a>(

src/controller/utils/text_to_speech.rs

@@ -3,7 +3,7 @@ use mxlink::{MatrixLink, MessageResponseType};
 
 use tracing::Instrument;
 
-use crate::controller::utils::mime::get_file_extension;
+use crate::utils::mime::get_file_extension;
 use crate::{
     Bot,
     agent::{AgentInstance, AgentPurpose, ControllerTrait, provider::TextToSpeechParams},

src/conversation/llm/entity.rs

@@ -1,4 +1,8 @@
 use chrono::{DateTime, Utc};
+use mxlink::matrix_sdk::ruma::events::room::message::ImageMessageEventContent;
+use mxlink::mime::Mime;
+
+use crate::agent::provider::ImageSource;
 
 #[derive(Debug, Clone, PartialEq)]
 pub enum Author {
@@ -10,10 +14,58 @@ pub enum Author {
 #[derive(Debug, Clone)]
 pub struct Message {
     pub author: Author,
-    pub message_text: String,
     pub timestamp: DateTime<Utc>,
+    pub content: MessageContent,
 }
 
+#[derive(Debug, Clone)]
+pub struct ImageDetails {
+    pub event_content: ImageMessageEventContent,
+    pub mime: Mime,
+    pub data: Vec<u8>,
+}
+
+impl ImageDetails {
+    pub fn new(event_content: ImageMessageEventContent, mime: Mime, data: Vec<u8>) -> Self {
+        Self {
+            event_content,
+            mime,
+            data,
+        }
+    }
+
+    pub fn filename(&self) -> String {
+        self.event_content
+            .filename
+            .clone()
+            .unwrap_or(self.event_content.body.clone())
+    }
+}
+
+impl From<ImageDetails> for ImageSource {
+    fn from(value: ImageDetails) -> Self {
+        ImageSource::new(value.filename(), value.data.clone(), value.mime.clone())
+    }
+}
+
+#[derive(Debug, Clone)]
+pub enum MessageContent {
+    Text(String),
+    Image(ImageDetails),
+}
+
+impl PartialEq for MessageContent {
+    fn eq(&self, other: &Self) -> bool {
+        match (self, other) {
+            (MessageContent::Text(a), MessageContent::Text(b)) => a == b,
+            (MessageContent::Image(a), MessageContent::Image(b)) => {
+                // We can probably do better than this by inspecting `.event_conten1t.source`, but for now this is good enough.
+                a.filename() == b.filename()
+            }
+            _ => false,
+        }
+    }
+}
 #[derive(Debug)]
 pub struct Conversation {
     pub messages: Vec<Message>,
@@ -28,27 +80,32 @@ impl Conversation {
     pub fn combine_consecutive_messages(&self) -> Conversation {
         // We'll likely get fewer messages, but let's reserve the maximum we expect.
         let mut new_messages = Vec::with_capacity(self.messages.len());
-        let mut last_seen_author: Option<Author> = None;
+        let mut last_seen_text_from_author: Option<Author> = None;
 
         for message in &self.messages {
-            let Some(last_seen_author_clone) = last_seen_author.clone() else {
-                last_seen_author = Some(message.author.clone());
+            let MessageContent::Text(message_text_content) = &message.content else {
+                last_seen_text_from_author = None;
+                new_messages.push(message.clone());
+                continue;
+            };
+
+            let Some(last_seen_author_clone) = last_seen_text_from_author.clone() else {
+                last_seen_text_from_author = Some(message.author.clone());
                 new_messages.push(message.clone());
                 continue;
             };
 
             if message.author != last_seen_author_clone {
-                last_seen_author = Some(message.author.clone());
+                last_seen_text_from_author = Some(message.author.clone());
                 new_messages.push(message.clone());
                 continue;
             }
 
-            new_messages.last_mut().unwrap().message_text.push('\n');
-            new_messages
-                .last_mut()
-                .unwrap()
-                .message_text
-                .push_str(&message.message_text);
+            let last_message = new_messages.last_mut().unwrap();
+            if let MessageContent::Text(ref mut text) = last_message.content {
+                text.push('\n');
+                text.push_str(message_text_content);
+            }
         }
 
         Conversation {
@@ -65,48 +122,76 @@ impl Conversation {
 mod tests {
     use super::*;
     use chrono::{TimeZone, Utc};
+    use mxlink::matrix_sdk::ruma::OwnedMxcUri;
+    use mxlink::mime;
 
     #[test]
     fn combine_consecutive_messages() {
         let timestamp_1 = Utc.with_ymd_and_hms(2024, 9, 20, 18, 34, 15).unwrap();
 
-        let timestamp_2 = Utc.with_ymd_and_hms(2024, 9, 21, 18, 34, 15).unwrap();
+        let timestamp_2 = Utc.with_ymd_and_hms(2024, 9, 21, 18, 34, 16).unwrap();
 
-        let timestamp_3 = Utc.with_ymd_and_hms(2024, 9, 22, 18, 34, 15).unwrap();
+        let timestamp_3 = Utc.with_ymd_and_hms(2024, 9, 22, 18, 34, 17).unwrap();
+
+        let timestamp_4 = Utc.with_ymd_and_hms(2024, 9, 23, 18, 34, 18).unwrap();
+
+        let image_event_content = ImageMessageEventContent::plain(
+            "image.png".to_string(),
+            OwnedMxcUri::from("mxc://example.com/1234567890"),
+        );
 
         let conversation = Conversation {
             messages: vec![
                 // User's turn
                 Message {
                     author: Author::User,
-                    message_text: "Hello".to_string(),
+                    content: MessageContent::Text("Hello".to_string()),
                     timestamp: timestamp_1,
                 },
                 Message {
                     author: Author::User,
-                    message_text: "How are you?".to_string(),
+                    content: MessageContent::Text("How are you?".to_string()),
                     timestamp: timestamp_2,
                 },
                 Message {
                     author: Author::User,
-                    message_text: "I'm OK, btw.".to_string(),
+                    content: MessageContent::Text("I'm OK, btw.".to_string()),
                     timestamp: timestamp_3,
                 },
+                Message {
+                    author: Author::User,
+                    content: MessageContent::Image(ImageDetails::new(
+                        image_event_content.clone(),
+                        mime::IMAGE_PNG,
+                        vec![],
+                    )),
+                    timestamp: timestamp_4,
+                },
+                Message {
+                    author: Author::User,
+                    content: MessageContent::Text("Above is an image.".to_string()),
+                    timestamp: timestamp_4,
+                },
+                Message {
+                    author: Author::User,
+                    content: MessageContent::Text("Would you take a look at it?".to_string()),
+                    timestamp: timestamp_4,
+                },
                 // Assistant's turn
                 Message {
                     author: Author::Assistant,
-                    message_text: "Hi there!".to_string(),
+                    content: MessageContent::Text("Hi there!".to_string()),
                     timestamp: timestamp_2,
                 },
                 Message {
                     author: Author::Assistant,
-                    message_text: "I'm doing well, thank you.".to_string(),
+                    content: MessageContent::Text("I'm doing well, thank you.".to_string()),
                     timestamp: timestamp_3,
                 },
                 // User's turn
                 Message {
                     author: Author::User,
-                    message_text: "That's great!".to_string(),
+                    content: MessageContent::Text("That's great!".to_string()),
                     timestamp: timestamp_3,
                 },
             ],
@@ -114,23 +199,44 @@ mod tests {
 
         let conversation = conversation.combine_consecutive_messages();
 
-        assert_eq!(conversation.messages.len(), 3);
+        assert_eq!(conversation.messages.len(), 5);
+
         assert_eq!(conversation.messages[0].author, Author::User);
         assert_eq!(
-            conversation.messages[0].message_text,
-            "Hello\nHow are you?\nI'm OK, btw."
+            conversation.messages[0].content,
+            MessageContent::Text("Hello\nHow are you?\nI'm OK, btw.".to_string())
         );
         assert_eq!(conversation.messages[0].timestamp, timestamp_1);
 
-        assert_eq!(conversation.messages[1].author, Author::Assistant);
+        assert_eq!(conversation.messages[1].author, Author::User);
         assert_eq!(
-            conversation.messages[1].message_text,
-            "Hi there!\nI'm doing well, thank you."
+            conversation.messages[1].content,
+            MessageContent::Image(ImageDetails::new(
+                image_event_content.clone(),
+                mime::IMAGE_PNG,
+                vec![],
+            ))
         );
-        assert_eq!(conversation.messages[1].timestamp, timestamp_2);
 
         assert_eq!(conversation.messages[2].author, Author::User);
-        assert_eq!(conversation.messages[2].message_text, "That's great!");
-        assert_eq!(conversation.messages[2].timestamp, timestamp_3);
+        assert_eq!(
+            conversation.messages[2].content,
+            MessageContent::Text("Above is an image.\nWould you take a look at it?".to_string())
+        );
+        assert_eq!(conversation.messages[2].timestamp, timestamp_4);
+
+        assert_eq!(conversation.messages[3].author, Author::Assistant);
+        assert_eq!(
+            conversation.messages[3].content,
+            MessageContent::Text("Hi there!\nI'm doing well, thank you.".to_string())
+        );
+        assert_eq!(conversation.messages[3].timestamp, timestamp_2);
+
+        assert_eq!(conversation.messages[4].author, Author::User);
+        assert_eq!(
+            conversation.messages[4].content,
+            MessageContent::Text("That's great!".to_string())
+        );
+        assert_eq!(conversation.messages[4].timestamp, timestamp_3);
     }
 }

src/conversation/llm/tests.rs

@@ -12,8 +12,7 @@ fn test_messages_by_the_bot_are_identified_correctly() {
 
     let matrix_message = super::super::matrix::MatrixMessage {
         sender_id: bot_user_id.to_owned(),
-        message_type: super::super::matrix::MatrixMessageType::Text,
-        message_text: "Hello!".to_owned(),
+        content: super::super::matrix::MatrixMessageContent::Text("Hello!".to_owned()),
         mentioned_users: vec![],
         timestamp: chrono::Utc::now(),
     };
@@ -21,7 +20,10 @@ fn test_messages_by_the_bot_are_identified_correctly() {
     let llm_message = convert_matrix_message_to_llm_message(&matrix_message, &bot_user_id).unwrap();
 
     assert_eq!(llm_message.author, Author::Assistant);
-    assert_eq!(llm_message.message_text, "Hello!");
+    assert_eq!(
+        llm_message.content,
+        MessageContent::Text("Hello!".to_string())
+    );
 }
 
 #[test]
@@ -37,8 +39,7 @@ fn test_notice_messages_by_bot_with_speech_to_text_prefix_are_cleaned_up_and_con
 
     let matrix_message = super::super::matrix::MatrixMessage {
         sender_id: bot_user_id.to_owned(),
-        message_type: super::super::matrix::MatrixMessageType::Notice,
-        message_text,
+        content: super::super::matrix::MatrixMessageContent::Notice(message_text),
         mentioned_users: vec![],
         timestamp: chrono::Utc::now(),
     };
@@ -46,7 +47,10 @@ fn test_notice_messages_by_bot_with_speech_to_text_prefix_are_cleaned_up_and_con
     let llm_message = convert_matrix_message_to_llm_message(&matrix_message, &bot_user_id).unwrap();
 
     assert_eq!(llm_message.author, Author::User);
-    assert_eq!(llm_message.message_text, source_message_text);
+    assert_eq!(
+        llm_message.content,
+        MessageContent::Text(source_message_text.to_string())
+    );
 }
 
 #[test]
@@ -61,8 +65,7 @@ fn test_notice_error_messages_by_bot_are_ignored() {
 
     let matrix_message = super::super::matrix::MatrixMessage {
         sender_id: bot_user_id.to_owned(),
-        message_type: super::super::matrix::MatrixMessageType::Notice,
-        message_text,
+        content: super::super::matrix::MatrixMessageContent::Notice(message_text),
         mentioned_users: vec![],
         timestamp: chrono::Utc::now(),
     };
@@ -86,8 +89,7 @@ fn test_other_notice_messages_by_the_bot_are_ignored() {
 
     let matrix_message = super::super::matrix::MatrixMessage {
         sender_id: bot_user_id.to_owned(),
-        message_type: super::super::matrix::MatrixMessageType::Notice,
-        message_text: message_text.to_owned(),
+        content: super::super::matrix::MatrixMessageContent::Notice(message_text.to_owned()),
         mentioned_users: vec![],
         timestamp: chrono::Utc::now(),
     };

src/conversation/llm/tokenization.rs

@@ -2,7 +2,7 @@ use tiktoken_rs::CoreBPE;
 use tiktoken_rs::get_bpe_from_tokenizer;
 use tiktoken_rs::tokenizer;
 
-use super::{Author, Message};
+use super::{Author, Message, MessageContent};
 
 fn get_bpe_for_model(model: &str) -> CoreBPE {
     let tokenizer = tokenizer::get_tokenizer(model)
@@ -71,7 +71,10 @@ fn calculate_token_size_for_message(bpe: &CoreBPE, model: &str, message: &Messag
         Author::Prompt => bpe.encode_with_special_tokens("system").len() as i32,
     };
 
-    let text_length = bpe.encode_with_special_tokens(&message.message_text).len() as i32;
+    let text_length = match &message.content {
+        MessageContent::Text(text) => bpe.encode_with_special_tokens(text).len() as i32,
+        MessageContent::Image(..) => 0,
+    };
 
     (text_length + role_length + tokens_per_message + tokens_per_name) as u32
 }
@@ -85,7 +88,7 @@ pub mod test {
 
         let message = super::Message {
             author: super::Author::User,
-            message_text: "Hello there!".to_owned(),
+            content: super::MessageContent::Text("Hello there!".to_string()),
             timestamp: chrono::Utc::now(),
         };
 
@@ -104,7 +107,7 @@ pub mod test {
 
         let prompt = super::Message {
             author: super::Author::Prompt,
-            message_text: "You are a bot!".to_owned(),
+            content: super::MessageContent::Text("You are a bot!".to_string()),
             timestamp: chrono::Utc::now(),
         };
         let prompt_length = 10;
@@ -118,7 +121,7 @@ pub mod test {
 
         let first = super::Message {
             author: super::Author::User,
-            message_text: "Hello there!".to_owned(),
+            content: super::MessageContent::Text("Hello there!".to_string()),
             timestamp: chrono::Utc::now(),
         };
         let first_length = 8;
@@ -132,7 +135,7 @@ pub mod test {
 
         let second = super::Message {
             author: super::Author::Assistant,
-            message_text: "Hello!".to_owned(),
+            content: super::MessageContent::Text("Hello!".to_string()),
             timestamp: chrono::Utc::now(),
         };
         let second_length = 7;
@@ -146,8 +149,9 @@ pub mod test {
 
         let third = super::Message {
             author: super::Author::User,
-            message_text: "This is the 3rd message in this conversation. It shall be preserved."
-                .to_owned(),
+            content: super::MessageContent::Text(
+                "This is the 3rd message in this conversation. It shall be preserved.".to_owned(),
+            ),
             timestamp: chrono::Utc::now(),
         };
         let third_length = 21;
@@ -161,7 +165,9 @@ pub mod test {
 
         let forth = super::Message {
             author: super::Author::Assistant,
-            message_text: "This is yet another message that shall be preserved.".to_owned(),
+            content: super::MessageContent::Text(
+                "This is yet another message that shall be preserved.".to_owned(),
+            ),
             timestamp: chrono::Utc::now(),
         };
         let forth_length = 15;
@@ -186,13 +192,13 @@ pub mod test {
         assert_eq!(2, new_conversation_messages.len());
 
         assert_eq!(
-            new_conversation_messages.first().unwrap().message_text,
-            third.message_text
+            new_conversation_messages.first().unwrap().content,
+            third.content
         );
 
         assert_eq!(
-            new_conversation_messages.last().unwrap().message_text,
-            forth.message_text
+            new_conversation_messages.last().unwrap().content,
+            forth.content
         );
     }
 
@@ -206,7 +212,7 @@ pub mod test {
 
         let prompt = super::Message {
             author: super::Author::User,
-            message_text: "あなたはボットです。".to_owned(),
+            content: super::MessageContent::Text("あなたはボットです。".to_string()),
             timestamp: chrono::Utc::now(),
         };
         let prompt_length = 14;
@@ -220,7 +226,7 @@ pub mod test {
 
         let first = super::Message {
             author: super::Author::User,
-            message_text: "こんにちは!".to_owned(),
+            content: super::MessageContent::Text("こんにちは!".to_string()),
             timestamp: chrono::Utc::now(),
         };
         let first_length = 7;
@@ -234,7 +240,7 @@ pub mod test {
 
         let second = super::Message {
             author: super::Author::Assistant,
-            message_text: "こんにちは。今日は元気ですか。".to_owned(),
+            content: super::MessageContent::Text("こんにちは。今日は元気ですか。".to_string()),
             timestamp: chrono::Utc::now(),
         };
         let second_length = 15;
@@ -248,7 +254,9 @@ pub mod test {
 
         let third = super::Message {
             author: super::Author::User,
-            message_text: "これは第3のメッセージなので、保存されます。".to_owned(),
+            content: super::MessageContent::Text(
+                "これは第3のメッセージなので、保存されます。".to_string(),
+            ),
             timestamp: chrono::Utc::now(),
         };
         let third_length = 22;
@@ -262,7 +270,9 @@ pub mod test {
 
         let forth = super::Message {
             author: super::Author::Assistant,
-            message_text: "これはもう一つの保存されますメッセージです。".to_owned(),
+            content: super::MessageContent::Text(
+                "これはもう一つの保存されますメッセージです。".to_string(),
+            ),
             timestamp: chrono::Utc::now(),
         };
         let forth_length = 21;
@@ -287,13 +297,13 @@ pub mod test {
         assert_eq!(2, new_conversation_messages.len());
 
         assert_eq!(
-            new_conversation_messages.first().unwrap().message_text,
-            third.message_text
+            new_conversation_messages.first().unwrap().content,
+            third.content
         );
 
         assert_eq!(
-            new_conversation_messages.last().unwrap().message_text,
-            forth.message_text
+            new_conversation_messages.last().unwrap().content,
+            forth.content
         );
     }
 }

src/conversation/llm/utils.rs

@@ -1,7 +1,7 @@
-use matrix_sdk::ruma::OwnedUserId;
+use mxlink::matrix_sdk::ruma::OwnedUserId;
 
-use super::{Author, Message};
-use crate::conversation::matrix::{MatrixMessage, MatrixMessageType};
+use super::entity::{Author, ImageDetails, Message, MessageContent};
+use crate::conversation::matrix::{MatrixMessage, MatrixMessageContent};
 use crate::utils::text_to_speech as text_to_speech_utils;
 
 pub fn convert_matrix_message_to_llm_message(
@@ -16,13 +16,22 @@ pub fn convert_matrix_message_to_llm_message(
 }
 
 fn convert_bot_message(matrix_message: &MatrixMessage) -> Option<Message> {
-    match matrix_message.message_type {
-        MatrixMessageType::Text => {
-            convert_bot_text_message(&matrix_message.message_text, &matrix_message.timestamp)
+    match &matrix_message.content {
+        MatrixMessageContent::Text(text) => {
+            convert_bot_text_message(text, &matrix_message.timestamp)
         }
-        MatrixMessageType::Notice => {
-            convert_bot_notice_message(&matrix_message.message_text, &matrix_message.timestamp)
+        MatrixMessageContent::Notice(text) => {
+            convert_bot_notice_message(text, &matrix_message.timestamp)
         }
+        MatrixMessageContent::Image(image_content, mime_type, media_bytes) => Some(Message {
+            author: Author::Assistant,
+            content: MessageContent::Image(ImageDetails::new(
+                image_content.clone(),
+                mime_type.clone(),
+                media_bytes.clone(),
+            )),
+            timestamp: matrix_message.timestamp.to_owned(),
+        }),
     }
 }
 
@@ -32,7 +41,7 @@ fn convert_bot_text_message(
 ) -> Option<Message> {
     Some(Message {
         author: Author::Assistant,
-        message_text: text.to_owned(),
+        content: MessageContent::Text(text.to_owned()),
         timestamp: timestamp.to_owned(),
     })
 }
@@ -52,7 +61,7 @@ fn convert_bot_notice_message(
         // This is a transcription message. We remove the prefix and consider it as a message sent by the user.
         return Some(Message {
             author: Author::User,
-            message_text: text.to_owned(),
+            content: MessageContent::Text(text.to_owned()),
             timestamp: timestamp.to_owned(),
         });
     }
@@ -61,9 +70,25 @@ fn convert_bot_notice_message(
 }
 
 fn convert_user_message(matrix_message: &MatrixMessage) -> Option<Message> {
-    Some(Message {
-        author: Author::User,
-        message_text: matrix_message.message_text.clone(),
-        timestamp: matrix_message.timestamp.to_owned(),
-    })
+    match &matrix_message.content {
+        MatrixMessageContent::Text(text) => Some(Message {
+            author: Author::User,
+            content: MessageContent::Text(text.clone()),
+            timestamp: matrix_message.timestamp.to_owned(),
+        }),
+        MatrixMessageContent::Notice(text) => Some(Message {
+            author: Author::User,
+            content: MessageContent::Text(text.clone()),
+            timestamp: matrix_message.timestamp.to_owned(),
+        }),
+        MatrixMessageContent::Image(image_content, mime_type, media_bytes) => Some(Message {
+            author: Author::User,
+            content: MessageContent::Image(ImageDetails::new(
+                image_content.clone(),
+                mime_type.clone(),
+                media_bytes.clone(),
+            )),
+            timestamp: matrix_message.timestamp.to_owned(),
+        }),
+    }
 }

src/conversation/matrix/entity.rs

@@ -2,20 +2,22 @@ use chrono::{DateTime, Utc};
 use regex::Regex;
 
 use mxlink::matrix_sdk::ruma::OwnedUserId;
+use mxlink::matrix_sdk::ruma::events::room::message::ImageMessageEventContent;
+use mxlink::mime::Mime;
 
 #[derive(Clone)]
 pub struct MatrixMessage {
     pub sender_id: OwnedUserId,
-    pub message_type: MatrixMessageType,
-    pub message_text: String,
+    pub content: MatrixMessageContent,
     pub mentioned_users: Vec<OwnedUserId>,
     pub timestamp: DateTime<Utc>,
 }
 
 #[derive(Clone)]
-pub enum MatrixMessageType {
-    Text,
-    Notice,
+pub enum MatrixMessageContent {
+    Text(String),
+    Notice(String),
+    Image(ImageMessageEventContent, Mime, Vec<u8>),
 }
 
 #[derive(Clone)]

src/conversation/matrix/mod.rs

@@ -6,6 +6,6 @@ mod utils;
 pub(crate) use room_display_name_fetcher::RoomDisplayNameFetcher;
 pub(crate) use room_event_fetcher::RoomEventFetcher;
 
-pub(crate) use entity::{MatrixMessage, MatrixMessageProcessingParams, MatrixMessageType};
+pub(crate) use entity::{MatrixMessage, MatrixMessageContent, MatrixMessageProcessingParams};
 
 pub(crate) use utils::*;

src/conversation/matrix/utils/mod.rs

@@ -18,9 +18,11 @@ use mxlink::matrix_sdk::{
     },
 };
 use mxlink::{MatrixLink, ThreadGetMessagesParams, ThreadInfo};
+use tracing::Instrument;
 
-use super::{MatrixMessage, MatrixMessageProcessingParams, MatrixMessageType, RoomEventFetcher};
+use super::{MatrixMessage, MatrixMessageContent, MatrixMessageProcessingParams, RoomEventFetcher};
 use crate::entity::{InteractionContext, InteractionTrigger, MessagePayload};
+use crate::utils::mime::get_mime_type_from_file_name;
 
 struct DetailedMessagePayload {
     is_mentioning_bot: bool,
@@ -28,7 +30,7 @@ struct DetailedMessagePayload {
 }
 
 pub async fn get_matrix_messages_in_thread(
-    matrix_link: MatrixLink,
+    matrix_link: &MatrixLink,
     room: &Room,
     thread_id: OwnedEventId,
 ) -> Result<Vec<MatrixMessage>, mxlink::matrix_sdk::Error> {
@@ -40,18 +42,20 @@ pub async fn get_matrix_messages_in_thread(
     let mut messages: Vec<MatrixMessage> = Vec::new();
 
     for matrix_native_message in messages_native {
-        let Some(message) = convert_matrix_native_event_to_matrix_message(&matrix_native_message)
-        else {
-            continue;
-        };
+        let message_result =
+            convert_matrix_native_event_to_matrix_message(matrix_link, &matrix_native_message)
+                .await?;
 
-        messages.push(message);
+        if let Some(message) = message_result {
+            messages.push(message);
+        }
     }
 
     Ok(messages)
 }
 
 pub async fn get_matrix_messages_in_reply_chain(
+    matrix_link: &MatrixLink,
     event_fetcher: &Arc<RoomEventFetcher>,
     room: &Room,
     event_id: OwnedEventId,
@@ -62,12 +66,13 @@ pub async fn get_matrix_messages_in_reply_chain(
     let mut messages: Vec<MatrixMessage> = Vec::new();
 
     for matrix_native_message in messages_native {
-        let Some(message) = convert_matrix_native_event_to_matrix_message(&matrix_native_message)
-        else {
-            continue;
-        };
+        let message_result =
+            convert_matrix_native_event_to_matrix_message(matrix_link, &matrix_native_message)
+                .await?;
 
-        messages.push(message);
+        if let Some(message) = message_result {
+            messages.push(message);
+        }
     }
 
     Ok(messages)
@@ -150,30 +155,34 @@ pub async fn process_matrix_messages(
         let mut message = message.clone();
 
         if i == 0 && !params.first_message_prefixes_to_strip.is_empty() {
-            let mut message_text = message.message_text.clone();
+            if let MatrixMessageContent::Text(message_text) = &message.content {
+                let mut message_text = message_text.clone();
 
-            for prefix in &params.first_message_prefixes_to_strip {
-                if let Some(message_text_stripped) = message_text.strip_prefix(prefix) {
-                    message_text = message_text_stripped.to_owned();
+                for prefix in &params.first_message_prefixes_to_strip {
+                    if let Some(message_text_stripped) = message_text.strip_prefix(prefix) {
+                        message_text = message_text_stripped.to_owned();
+                    }
                 }
-            }
 
-            message.message_text = message_text.trim().to_owned();
+                message.content = MatrixMessageContent::Text(message_text.trim().to_owned());
+            }
         }
 
         // We only strip `bot_user_prefixes_to_strip`-defined prefixes from messages that mention the bot user.
         if !params.bot_user_prefixes_to_strip.is_empty()
             && message.mentioned_users.contains(&params.bot_user_id)
         {
-            let mut message_text = message.message_text.clone();
+            if let MatrixMessageContent::Text(message_text) = &message.content {
+                let mut message_text = message_text.clone();
 
-            for prefix in &params.bot_user_prefixes_to_strip {
-                if let Some(message_text_stripped) = message_text.strip_prefix(prefix) {
-                    message_text = message_text_stripped.to_owned();
+                for prefix in &params.bot_user_prefixes_to_strip {
+                    if let Some(message_text_stripped) = message_text.strip_prefix(prefix) {
+                        message_text = message_text_stripped.to_owned();
+                    }
                 }
-            }
 
-            message.message_text = message_text.trim().to_owned();
+                message.content = MatrixMessageContent::Text(message_text.trim().to_owned());
+            }
         }
 
         messages_filtered.push(message);
@@ -207,23 +216,25 @@ fn is_message_from_allowed_sender(
     false
 }
 
-pub fn convert_matrix_native_event_to_matrix_message(
+pub async fn convert_matrix_native_event_to_matrix_message(
+    matrix_link: &MatrixLink,
     matrix_native_event: &AnySyncMessageLikeEvent,
-) -> Option<MatrixMessage> {
+) -> Result<Option<MatrixMessage>, mxlink::matrix_sdk::Error> {
     let Some(content) = matrix_native_event.original_content() else {
         // Redacted message
-        return None;
+        return Ok(None);
     };
 
     let AnyMessageLikeEventContent::RoomMessage(room_message) = content else {
         // Some state event, etc.
-        return None;
+        return Ok(None);
     };
 
     let (text, is_notice) = match &room_message.msgtype {
         MessageType::Text(text_content) => (text_content.body.clone(), false),
         MessageType::Notice(notice_content) => (notice_content.body.clone(), true),
-        _ => return None,
+        MessageType::Image(image_content) => (image_content.body.clone(), false),
+        _ => return Ok(None),
     };
 
     let is_reply = matches!(room_message.relates_to, Some(Relation::Reply { .. }));
@@ -248,17 +259,48 @@ pub fn convert_matrix_native_event_to_matrix_message(
         .map(|m| m.user_ids.iter().map(|u| u.to_owned()).collect())
         .unwrap_or(vec![]);
 
-    Some(MatrixMessage {
+    if let MessageType::Image(image_content) = &room_message.msgtype {
+        let media_request = mxlink::matrix_sdk::media::MediaRequestParameters {
+            source: image_content.source.to_owned(),
+            format: mxlink::matrix_sdk::media::MediaFormat::File,
+        };
+
+        let file_name = image_content
+            .filename
+            .clone()
+            .unwrap_or(image_content.body.clone());
+
+        let mime_type = get_mime_type_from_file_name(&file_name);
+
+        tracing::debug!("Determined mime type {} for file {}", mime_type, file_name);
+
+        let span = tracing::debug_span!("get_media_content", file_name = %file_name, mime_type = %mime_type);
+
+        let media_bytes = matrix_link
+            .client()
+            .media()
+            .get_media_content(&media_request, true)
+            .instrument(span)
+            .await?;
+
+        return Ok(Some(MatrixMessage {
+            sender_id: matrix_native_event.sender().to_owned(),
+            content: MatrixMessageContent::Image(image_content.clone(), mime_type, media_bytes),
+            mentioned_users,
+            timestamp,
+        }));
+    }
+
+    Ok(Some(MatrixMessage {
         sender_id: matrix_native_event.sender().to_owned(),
-        message_type: if is_notice {
-            MatrixMessageType::Notice
+        content: if is_notice {
+            MatrixMessageContent::Notice(text)
         } else {
-            MatrixMessageType::Text
+            MatrixMessageContent::Text(text)
         },
-        message_text: text,
         mentioned_users,
         timestamp,
-    })
+    }))
 }
 
 /// Determines the interaction context for an incoming (new) room event.

src/conversation/matrix/utils/tests.rs

@@ -3,7 +3,7 @@ use chrono::{TimeZone, Utc};
 use mxlink::matrix_sdk::ruma::OwnedUserId;
 
 use crate::conversation::matrix::{
-    MatrixMessage, MatrixMessageProcessingParams, MatrixMessageType,
+    MatrixMessage, MatrixMessageContent, MatrixMessageProcessingParams,
 };
 
 #[test]
@@ -17,24 +17,21 @@ fn is_message_from_allowed_sender() {
 
     let bot_message = MatrixMessage {
         sender_id: bot_user_id.to_owned(),
-        message_type: MatrixMessageType::Text,
-        message_text: "Hello!".to_owned(),
+        content: MatrixMessageContent::Text("Hello!".to_owned()),
         mentioned_users: vec![],
         timestamp,
     };
 
     let allowed_user_message = MatrixMessage {
         sender_id: allowed_user_id.to_owned(),
-        message_type: MatrixMessageType::Text,
-        message_text: "Hello!".to_owned(),
+        content: MatrixMessageContent::Text("Hello!".to_owned()),
         mentioned_users: vec![],
         timestamp,
     };
 
     let unallowed_user_message = MatrixMessage {
         sender_id: unallowed_user_id.to_owned(),
-        message_type: MatrixMessageType::Text,
-        message_text: "Hello!".to_owned(),
+        content: MatrixMessageContent::Text("Hello!".to_owned()),
         mentioned_users: vec![],
         timestamp,
     };
@@ -88,48 +85,42 @@ async fn process_matrix_messages() {
 
     let allowed_user_message = MatrixMessage {
         sender_id: allowed_user_id.to_owned(),
-        message_type: MatrixMessageType::Text,
-        message_text: "Hello from the user!".to_owned(),
+        content: MatrixMessageContent::Text("Hello from the user!".to_owned()),
         mentioned_users: vec![],
         timestamp,
     };
 
     let allowed_user_message_with_prefix = MatrixMessage {
         sender_id: allowed_user_id.to_owned(),
-        message_type: MatrixMessageType::Text,
-        message_text: "!bai Hello from the user!".to_owned(),
+        content: MatrixMessageContent::Text("!bai Hello from the user!".to_owned()),
         mentioned_users: vec![],
         timestamp,
     };
 
     let allowed_user_message_with_prefix_no_space = MatrixMessage {
         sender_id: allowed_user_id.to_owned(),
-        message_type: MatrixMessageType::Text,
-        message_text: "!baiHello from the user!".to_owned(),
+        content: MatrixMessageContent::Text("!baiHello from the user!".to_owned()),
         mentioned_users: vec![],
         timestamp,
     };
 
     let allowed_user_message_with_prefix_full_width_space = MatrixMessage {
         sender_id: allowed_user_id.to_owned(),
-        message_type: MatrixMessageType::Text,
-        message_text: "!bai　Hello from the user!".to_owned(),
+        content: MatrixMessageContent::Text("!bai　Hello from the user!".to_owned()),
         mentioned_users: vec![],
         timestamp,
     };
 
     let bot_message = MatrixMessage {
         sender_id: bot_user_id.to_owned(),
-        message_type: MatrixMessageType::Text,
-        message_text: "Hello from the bot!".to_owned(),
+        content: MatrixMessageContent::Text("Hello from the bot!".to_owned()),
         mentioned_users: vec![],
         timestamp,
     };
 
     let allowed_user_message_with_bot_mention = MatrixMessage {
         sender_id: allowed_user_id.to_owned(),
-        message_type: MatrixMessageType::Text,
-        message_text: "@baibot: Hello from the user!".to_owned(),
+        content: MatrixMessageContent::Text("@baibot: Hello from the user!".to_owned()),
         mentioned_users: vec![bot_user_id.to_owned()],
         timestamp,
     };
@@ -137,16 +128,14 @@ async fn process_matrix_messages() {
     // The message text is the same as above - it mentions the bot, but the actually-mentioned user is another user.
     let allowed_user_message_with_another_user_mention = MatrixMessage {
         sender_id: allowed_user_id.to_owned(),
-        message_type: MatrixMessageType::Text,
-        message_text: allowed_user_message_with_bot_mention.message_text.clone(),
+        content: allowed_user_message_with_bot_mention.content.clone(),
         mentioned_users: vec![allowed_user_id.to_owned()],
         timestamp,
     };
 
     let unallowed_user_message = MatrixMessage {
         sender_id: unallowed_user_id.to_owned(),
-        message_type: MatrixMessageType::Text,
-        message_text: "Hello from an unallowed user!".to_owned(),
+        content: MatrixMessageContent::Text("Hello from an unallowed user!".to_owned()),
         mentioned_users: vec![],
         timestamp,
     };
@@ -285,7 +274,10 @@ async fn process_matrix_messages() {
 
         let processed_message_texts = processed_messages
             .iter()
-            .map(|message| message.message_text.clone())
+            .map(|message| match &message.content {
+                MatrixMessageContent::Text(text) => text.clone(),
+                _ => "".to_owned(),
+            })
             .collect::<Vec<String>>();
 
         assert_eq!(

src/conversation/matrix_llm_bridge.rs

@@ -12,7 +12,7 @@ use super::matrix::{
 };
 
 pub async fn create_llm_conversation_for_matrix_thread(
-    matrix_link: MatrixLink,
+    matrix_link: &MatrixLink,
     room: &mxlink::matrix_sdk::Room,
     thread_id: OwnedEventId,
     params: &MatrixMessageProcessingParams,
@@ -27,12 +27,14 @@ pub async fn create_llm_conversation_for_matrix_thread(
 }
 
 pub async fn create_llm_conversation_for_matrix_reply_chain(
+    matrix_link: &MatrixLink,
     event_fetcher: &Arc<RoomEventFetcher>,
     room: &mxlink::matrix_sdk::Room,
     event_id: OwnedEventId,
     params: &MatrixMessageProcessingParams,
 ) -> Result<Conversation, mxlink::matrix_sdk::Error> {
-    let messages = get_matrix_messages_in_reply_chain(event_fetcher, room, event_id).await?;
+    let messages =
+        get_matrix_messages_in_reply_chain(matrix_link, event_fetcher, room, event_id).await?;
 
     let llm_messages = filter_messages_and_convert_to_llm_messages(messages, params).await;
 

src/entity/message_payload.rs

@@ -1,5 +1,5 @@
 use mxlink::matrix_sdk::ruma::events::room::message::{
-    AudioMessageEventContent, MessageType, TextMessageEventContent,
+    AudioMessageEventContent, ImageMessageEventContent, MessageType, TextMessageEventContent,
 };
 use mxlink::matrix_sdk::ruma::{OwnedEventId, OwnedUserId};
 
@@ -29,6 +29,7 @@ pub enum MessagePayload {
 
     Text(TextMessageEventContent),
     Audio(AudioMessageEventContent),
+    Image(ImageMessageEventContent),
 
     Reaction {
         key: String,
@@ -55,6 +56,7 @@ impl TryInto<MessagePayload> for MessageType {
                 // For this reason, we handle all audio.
                 MessagePayload::Audio(audio_content)
             }
+            MessageType::Image(image_content) => MessagePayload::Image(image_content),
             other => {
                 return Err(format!("Unsupported message type: {:?}", other));
             }

src/strings/image_edit.rs

@@ -0,0 +1,13 @@
+pub fn guide_how_to_proceed() -> String {
+    let mut message = String::new();
+
+    message.push_str("💡 Respond in this thread (in any order) with:\n");
+    message.push_str("- one or more images: to use the given images for creating an edit\n");
+    message.push_str("- more messages: to expand on your original prompt\n");
+    message.push_str("- a message saying `go`: to generate an edit with the current prompt\n");
+    message.push_str(
+        "- a message saying `again`: to generate one more image edit with the current prompt\n",
+    );
+
+    message
+}

src/strings/image_generation.rs

@@ -8,7 +8,7 @@ pub fn guide_how_to_proceed() -> String {
     message.push_str("💡 Respond in this thread with:\n");
     message.push_str("- more messages: to expand on your original prompt\n");
     message.push_str(
-        "- a message saying `again`: to generate one more image with the current prompt.\n",
+        "- a message saying `again`: to generate one more image with the current prompt\n",
     );
 
     message

src/strings/mod.rs

@@ -4,6 +4,7 @@ pub mod cfg;
 pub mod error;
 pub mod global_config;
 pub mod help;
+pub mod image_edit;
 pub mod image_generation;
 pub mod introduction;
 pub mod provider;

src/strings/provider.rs

@@ -1,6 +1,7 @@
 use crate::agent::AgentInstantiationError;
 use crate::agent::AgentProvider;
 use crate::agent::AgentProviderInfo;
+use crate::agent::AgentPurpose;
 
 pub fn invalid(provider: &str) -> String {
     let choices_string = AgentProvider::choices()
@@ -105,7 +106,17 @@ pub fn help_provider_details(id: &str, info: &AgentProviderInfo) -> String {
 
     let mut capabilities = vec![];
     for purpose in info.supported_purposes.iter() {
-        capabilities.push(format!("{} {}", purpose.emoji(), purpose.as_str()));
+        let mut purpose_line = format!("{} {}", purpose.emoji(), purpose.as_str());
+
+        if let AgentPurpose::TextGeneration = purpose {
+            if info.text_generation_supports_vision {
+                purpose_line = format!("{} ({})", purpose_line, "incl. vision");
+            } else {
+                purpose_line = format!("{} ({})", purpose_line, "no vision");
+            }
+        }
+
+        capabilities.push(purpose_line);
     }
 
     message.push_str(&format!("- 🌟 Capabilities: {}\n", capabilities.join(", ")));

src/strings/usage.rs

@@ -34,26 +34,37 @@ By default, the bot will also perform 💬 Text Generation on the text. This is
 If all your messages are in the same language, you can improve accuracy & latency by configuring the language via the **🦻 Speech-to-Text / 🔤 Language** setting.
 
 
-### 🖌️ Image Generation
+### Image Generation
 
-#### Generating images
+#### 🖌️ Creating images
 
-Simply send a command like `%command_prefix% image A beautiful sunset over the ocean` and the bot will start a threaded conversation and post an image based on your prompt.
+Simply send a command like `%command_prefix% image create A beautiful sunset over the ocean` and the bot will start a threaded conversation and post an image based on your prompt.
 
-You can then, respond in the same message thread with:
+You can then respond in the same message thread with:
 
 - more messages, to add more criteria to your prompt.
 - a message saying `again`, to generate one more image with the current prompt.
 
-#### Generating stickers
+#### 🎨 Editing images
 
-A variation of **generating images** is to generate "sticker images".
+Simply send a command like `%command_prefix% image edit Turn the following image into an anime-style drawing` and the bot will start a threaded conversation asking for more details.
 
-To generate a sticker, send a command like `%command_prefix% sticker A huge bowl of steaming ramen with a mountain of beansprouts on top`.
+You can then respond in the same message thread with:
 
-The difference from **generating images** is that the bot will:
+- more messages, to add more criteria to your prompt.
+- one or more images, to provide the images that the bot will operate on.
+- a message saying `go`, to start the image generation process.
+- a message saying `again`, to prompt the bot to generate one more image edit with the current prompt.
 
-- generate a smaller-resolution image (`256x256`) - smaller/quicker, but still good enough for a sticker
+#### 🫵 Creating stickers
+
+A variation of **creating images** is creating "sticker images".
+
+To create a sticker, send a command like `%command_prefix% sticker A huge bowl of steaming ramen with a mountain of beansprouts on top`.
+
+The difference from **creating images** is that the bot will:
+
+- create a smaller-resolution image (`256x256`) - smaller/quicker, but still good enough for a sticker
 - potentially switch to a different (cheaper or otherwise more suitable) model, if available
 - post the image directly to the room (as a reply to your message), without starting a threaded conversation
 "#;

src/utils/base64.rs

@@ -0,0 +1,9 @@
+use base64::{Engine as _, engine::general_purpose::STANDARD};
+
+pub(crate) fn base64_decode(base64_string: &str) -> Result<Vec<u8>, base64::DecodeError> {
+    STANDARD.decode(base64_string)
+}
+
+pub(crate) fn base64_encode(data: &[u8]) -> String {
+    STANDARD.encode(data)
+}

src/utils/mime.rs

@@ -0,0 +1,34 @@
+use mxlink::mime;
+
+pub fn get_file_extension(mime_type: &mime::Mime) -> String {
+    match (mime_type.type_(), mime_type.subtype()) {
+        (mime::AUDIO, mime::BASIC) => "au",
+        (mime::AUDIO, mime::MPEG) => "mp3",
+        (mime::AUDIO, mime::MP4) => "m4a",
+        (mime::AUDIO, mime::OGG) => "ogg",
+        (mime::IMAGE, mime::BMP) => "bmp",
+        (mime::IMAGE, mime::GIF) => "gif",
+        (mime::IMAGE, mime::JPEG) => "jpg",
+        (mime::IMAGE, mime::PNG) => "png",
+        (mime::IMAGE, mime::SVG) => "svg",
+        _ => "bin",
+    }
+    .to_string()
+}
+
+pub fn get_mime_type_from_file_name(file_name: &str) -> mime::Mime {
+    let extension = file_name.rsplit('.').next().unwrap_or("");
+
+    match extension.to_lowercase().as_str() {
+        "jpg" | "jpeg" => mime::IMAGE_JPEG,
+        "png" => mime::IMAGE_PNG,
+        "gif" => mime::IMAGE_GIF,
+        "webp" => "image/webp".parse().unwrap(),
+        "svg" => mime::IMAGE_SVG,
+        "tiff" | "tif" => "image/tiff".parse().unwrap(),
+        "bmp" => "image/bmp".parse().unwrap(),
+        "heic" | "heif" => "image/heic".parse().unwrap(),
+        "avif" => "image/avif".parse().unwrap(),
+        _ => mime::APPLICATION_OCTET_STREAM,
+    }
+}

src/utils/mod.rs

@@ -1,3 +1,5 @@
+pub(crate) mod base64;
+pub(crate) mod mime;
 pub mod status;
 pub mod text;
 pub mod text_to_speech;