Use conventional mxlink version requirement

Replace the unconventional wildcard lower-bound expression with a standard semver lower bound for readability and tooling consistency.

.github/workflows/workflow.yml

@@ -3,31 +3,62 @@ on:
   push:
     branches: [ "main" ]
     tags: [ "v*" ]
-  schedule:
-    - cron: '0 0 * * 1'
 permissions:
   checks: write
   contents: write
   packages: write
   pull-requests: read
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
 jobs:
   test-and-clippy:
     name: Unit testing and linting
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - uses: dtolnay/rust-toolchain@stable
+      - name: Install SQLite3
+        run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev
       - run: cargo test --all-features
       - run: cargo clippy
 
-  build-publish:
-    name: Build and Publish
-    runs-on: self-hosted
+  docker-clean-metadata:
+    runs-on: ubuntu-latest
+    outputs:
+      json: ${{ steps.meta.outputs.json }}
     steps:
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Login to ghcr.io
-        uses: docker/login-action@v3
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ghcr.io/${{ github.repository }}
+          tags: |
+            type=raw,value=latest,enable=${{ github.ref_name == 'main' }}
+            type=semver,pattern={{raw}}
+
+  docker-build:
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+    strategy:
+      matrix:
+        include:
+          - os: self-hosted
+            arch: amd64
+          - os: ubuntu-24.04-arm
+            arch: arm64
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+      - name: Log in to the GitHub Container registry
+        uses: docker/login-action@v4
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -36,16 +67,41 @@ jobs:
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: |
-            ghcr.io/${{ github.repository }}
-            registry.etke.cc/${{ github.repository }}
           tags: |
             type=raw,value=latest,enable=${{ github.ref_name == 'main' }}
             type=semver,pattern={{raw}}
-      - name: Build and push
+          flavor: |
+            latest=auto
+            suffix=-${{ matrix.arch }},onlatest=true
+          images: |
+            ghcr.io/${{ github.repository }}
+
+      - name: Build and push Docker images
         uses: docker/build-push-action@v6
         with:
-          platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+
+  docker-manifest:
+    needs:
+      - docker-build
+      - docker-clean-metadata
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        image: ${{ fromJson(needs.docker-clean-metadata.outputs.json).tags }}
+
+    steps:
+      - name: Log in to the GitHub Container registry
+        uses: docker/login-action@v4
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Create and push manifest
+        run: |
+          docker manifest create ${{ matrix.image }} ${{ matrix.image }}-amd64 ${{ matrix.image }}-arm64
+          docker manifest push ${{ matrix.image }}

.pre-commit-config.yaml

@@ -0,0 +1,36 @@
+repos:
+  # Fast built-in hooks (Rust-native, no dependencies)
+  - repo: builtin
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-merge-conflict
+      - id: check-added-large-files
+        args: ['--maxkb=1024']
+
+  # Local hooks that run project-specific tools
+  - repo: local
+    hooks:
+      - id: cargo-fmt-check
+        name: Cargo Format Check
+        entry: cargo fmt --all -- --check
+        language: system
+        files: '\.rs$'
+        pass_filenames: false
+
+      - id: cargo-clippy
+        name: Cargo Clippy
+        entry: cargo clippy -- -D warnings
+        language: system
+        files: '\.rs$'
+        pass_filenames: false
+        priority: 100
+
+      - id: test-unit
+        name: Unit Tests
+        entry: just test
+        language: system
+        files: '\.rs$'
+        pass_filenames: false
+        priority: 100

CHANGELOG.md

@@ -1 +1,251 @@
-There's nothing here yet.
+# (2026-02-18) Version 1.14.3
+
+- (**Internal Improvement**) Add [Renovate](https://docs.renovatebot.com/) configuration for automated dependency updates
+
+- (**Internal Improvement**) Dependency updates
+
+
+# (2026-02-18) Version 1.14.2
+
+- (**Internal Improvement**) Dependency updates
+
+- (**Internal Improvement**) Reorganize the development environment to support [Continuwuity](https://continuwuity.org/) as a homeserver choice (in addition to [Synapse](https://github.com/element-hq/synapse)). Continuwuity is now the default for its lighter footprint (no external database required). See [development docs](./docs/development.md) for details.
+
+
+# (2026-02-10) Version 1.14.1
+
+- (**Security**) Dependency updates to fix security vulnerabilities ([time](https://crates.io/crates/time) stack exhaustion DoS, [bytes](https://crates.io/crates/bytes) integer overflow), via [mxlink](https://crates.io/crates/mxlink) 1.12.0
+
+- (**Internal Improvement**) Switch from deprecated [serde_yaml](https://crates.io/crates/serde_yaml) to its maintained fork [serde_yaml_ng](https://crates.io/crates/serde_yaml_ng)
+
+- (**Internal Improvement**) Add [prek](https://github.com/nicholasgasior/prek) pre-commit hooks via [mise](https://mise.jdx.dev/) for automated code quality checks (formatting, clippy, tests)
+
+- (**Internal Improvement**) Fix clippy warnings and formatting issues
+
+
+# (2026-02-04) Version 1.14.0
+
+- (**Feature**) The `openai` provider now uses OpenAI's [Responses API](https://platform.openai.com/docs/api-reference/responses) (instead of the older Chat Completions API), adding support for [🛠️ built-in tools](./docs/features.md#️-built-in-tools-openai-only) (`web_search` and `code_interpreter`). These tools are **disabled by default** and can be enabled via the `text_generation.tools` configuration (see the [sample configuration](https://github.com/etkecc/baibot/blob/c70387b0c38d8d0f30bba2179a2a21a3710dbeaf/docs/sample-provider-configs/openai.yml#L12-L15)). To enable tools on an existing agent, you need to [update the agent](./docs/agents.md#updating-agents) to re-create it with the `text_generation.tools` section added and enable the tools you need. Thanks to [Layla Manley](https://github.com/yeslayla) for the contribution in [#62](https://github.com/etkecc/baibot/pull/62)!
+
+- (**Bugfix**) Fix sticker generation for newer GPT image models (`gpt-image-1`, `gpt-image-1-mini`, `gpt-image-1.5`) which don't support the previously hardcoded `256x256` size (minimum is `1024x1024`)
+
+- (**Internal Improvement**) Dependency updates
+
+
+# (2026-01-23) Version 1.13.0
+
+- (**Improvement**) Extend auto-switching to support cheaper models (`gpt-image-1-mini`) for `gpt-image-1` and `gpt-image-1.5` when generating stickers ([e0b4a40](https://github.com/etkecc/baibot/commit/e0b4a40))
+
+- (**Internal Improvement**) Upgrade Rust compiler (1.92.0 -> 1.93.0) ([691aeeb](https://github.com/etkecc/baibot/commit/691aeeb))
+
+- (**Internal Improvement**) Dependency updates
+
+
+# (2025-12-21) Version 1.12.0
+
+- (**Improvement**) Upgrade [async-openai](https://crates.io/crates/async-openai) (0.31.1 -> 0.32.2) and add support for OpenAI's `gpt-image-1.5` model ([08c689a](https://github.com/etkecc/baibot/commit/08c689a), [f7bf3d7](https://github.com/etkecc/baibot/commit/f7bf3d7))
+
+- (**Internal Improvement**) Dependency updates
+
+
+# (2025-12-15) Version 1.11.0
+
+- (**Feature**) Add support for custom avatars via file path and for keeping the already-set avatar (for those who wish to manage it by themselves via other means). See the [sample config](./etc/app/config.yml.dist) for details. ([062fbbb](https://github.com/etkecc/baibot/commit/062fbbb8ef9ad600db483a431c5c782402191023))
+
+- (**Internal Improvement**) Dependency updates ([99bde53](https://github.com/etkecc/baibot/commit/99bde53ef648a5a9086a96778fde4a9dbc1ede58))
+
+- (**Internal Improvement**) Documentation updates ([b3fd8e5](https://github.com/etkecc/baibot/commit/b3fd8e548f83fe46398ced4760d7e2bb7588c24d))
+
+- (**Internal Improvement**) Upgrade Rust compiler (1.91.1 -> 1.92.0) ([22906aa](https://github.com/etkecc/baibot/commit/22906aa2d3cae51815fad2560a545eaa69c247b6))
+
+
+# (2025-12-06) Version 1.10.0
+
+- (**Internal Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.11.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.16.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.16.0).
+
+# (2025-11-30) Version 1.9.0
+
+- (**Internal Improvement**) Upgrade [async-openai](https://crates.io/crates/async-openai) from our own etkecc fork (0.28.1-patched) to the official upstream version 0.31.1. This upgrade required some code adaptations to the new module structure, etc. While tested, regressions are possible.
+
+# (2025-11-28) Version 1.8.3
+
+- (**Improvement**) Add support for the `BAIBOT_PERSISTENCE_SESSION_ENCRYPTION_KEY` environment variable for configuring `persistence.session_encryption_key`
+
+- (**Improvement**) Add support for the `BAIBOT_USER_ENCRYPTION_RECOVERY_RESET_ALLOWED` environment variable for configuring `user.encryption.recovery_reset_allowed`
+
+- (**Internal Improvement**) Dependency updates.
+
+# (2025-11-20) Version 1.8.2
+
+- (**Internal Improvement**) Dependency and compiler updates (Rust 1.89.0 -> 1.91.1).
+
+# (2025-09-12) Version 1.8.1
+
+- (**Internal Improvement**) Dependency updates.
+
+# (2025-09-08) Version 1.8.0
+
+- (**Internal Improvement**) Upgrade [mxlink](https://crates.io/crates/mxlink) (1.9.0 -> 1.10.0) and [matrix-sdk](https://crates.io/crates/matrix-sdk) (0.13.0 -> 0.14.0)
+
+- (**Internal Improvement**) Upgrade [Rust](https://www.rust-lang.org/) (1.88.0 -> 1.89.0)
+
+- (**Internal Improvement**) Upgrade Debian base for container images (12/bookworm -> 13/trixie)
+
+# (2025-07-11) Version 1.7.6
+
+- (**Internal Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.9.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.13.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.13.0), which contains fixes for some security vulnerabilities)
+
+# (2025-06-10) Version 1.7.5
+
+- (**Internal Improvement**) Dependency and compiler updates (Rust 1.86 -> 1.86).
+
+# (2025-06-10) Version 1.7.4
+
+- (**Internal Improvement**) Dependency updates.
+
+# (2025-06-10) Version 1.7.3
+
+- (**Internal Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.8.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.12.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.12.0), which contains fixes for important security vulnerabilities)
+
+# (2025-05-11) Version 1.7.2
+
+- (**Bugfix**) Allow `image_generation.size` configuration value for OpenAI to be `null` to allow the model to choose the size automatically and default to that
+
+# (2025-05-11) Version 1.7.1
+
+- (**Bugfix**) Fix lack of documentation for the new [image-editing](./docs/features.md#-image-editing) feature in the `!bai usage` command's output
+
+# (2025-05-10) Version 1.7.0
+
+- (**Feature**) Add vision support to the OpenAI and Anthropic providers. You can now mix text and images in your conversations - fixes [issue #5](https://github.com/etkecc/baibot/issues/5)
+
+- (**Feature**) Add [image-editing](./docs/features.md#-image-editing) support to the OpenAI provider
+
+- (**Improvement**) Add compatibility with OpenAI's `gpt-image-1` model - fixes [issue #40](https://github.com/etkecc/baibot/issues/40)
+
+- (**Change**) Rework [image-creation](./docs/features.md#-image-creation) to avoid command conflicts with [image-editing](./docs/features.md#-image-editing). The image-creation command syntax is now `!bai image create <prompt>` (previously: `!bai image <prompt>`).
+
+- (**Internal Improvement**) Dependency and compiler updates
+
+> [!WARNING]
+> Unlike other releases, this release is not published to [crates.io](https://crates.io), because it relies on multiple library forks (`async-openai` and `anthropic-rs`) sourced from Github.
+
+
+# (2025-04-12) Version 1.6.0
+
+- (**Internal Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.7.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.11.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.11.0))
+
+
+# (2025-03-31) Version 1.5.1
+
+- (**Internal Improvement**) Dependency updates
+
+# (2025-02-27) Version 1.5.0
+
+- (**Feature**) Add support for sending Speech-to-Text replies for [Transcribe-only mode](./docs/features.md#transcribe-only-mode) as regular text messages instead of notices and doing it so by default ([a1bd292752](https://github.com/etkecc/baibot/commit/a1bd292752bdd37a196788c73d00b5619e843a78)) - improvement for [issue #14](https://github.com/etkecc/baibot/issues/14). See [🦻 Speech-to-Text / 🪄 Message Type for non-threaded only-transcribed messages](./docs/configuration/speech-to-text.md#-message-type-for-non-threaded-only-transcribed-messages) for details.
+
+- (**Feature**) Add config setting controlling if a self-introduction message is posted after joining a room ([c051da2f4a](https://github.com/etkecc/baibot/commit/c051da2f4a161de0974ebb917f7a52d01f5a001f)) - fixes [issue #32](https://github.com/etkecc/baibot/issues/32). You may wish to add a `room.post_join_self_introduction_enabled` property to your configuration. See the [sample config](./etc/app/config.yml.dist) for details. If unspecified, it defaults to `true` anyway which preserves the old behavior.
+
+- (**Feature**) Add support for configuring `max_completion_tokens` for OpenAI ([47d8edea70](https://github.com/etkecc/baibot/commit/47d8edea705a44aa25a9bfaec4888c0f9ea8700e))
+
+- (**Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.6.1 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.10.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.10.0))
+
+- (**Improvement**) Populate image/audio attachment `body` with a filename, not with text to avoid incorrect rendering in Element Web, etc. ([ec1879d212](https://github.com/etkecc/baibot/commit/ec1879d212fa8d6e5f8590486e94c72abfcb75a5))
+
+- (**Improvement**) Replace Anthropic library ([anthropic-rs](https://crates.io/crates/anthropic-rs) -> [anthropic](https://crates.io/crates/anthropic)) and switch default recommended model (`claude-3-5-sonnet-20240620` -> `claude-3-7-sonnet-20250219`) ([692d61b239](https://github.com/etkecc/baibot/commit/692d61b2398f073b81d32d4cbe8145ab3929e48c)) - fixes [issue #22](https://github.com/etkecc/baibot/issues/22)
+
+- (**Internal Improvement**) Switch to native building of `arm64` container images to decrease total build times from ~40 minutes to ~8 minutes ([6719538530b](https://github.com/etkecc/baibot/commit/6719538530bf76b3ff2d24077b2a7fa868276b79))
+
+- (**Internal Improvement**) Various other internal changes, including upgrading [Rust from 1.82 to 1.85 and switching to Rust edition 2024](https://blog.rust-lang.org/2025/02/20/Rust-1.85.0.html)
+
+
+# (2024-12-12) Version 1.4.1
+
+- (**Bugfix**) Fix detection for whether the bot is the last member in a room, to avoid incorrectly leaving multi-user rooms that have had at least one person `leave` ([3c47d40781](https://github.com/etkecc/baibot/commit/3c47d407819aa9c0121117a411858238724f06da))
+
+
+# (2024-11-19) Version 1.4.0
+
+- (**Improvement**) Dependency updates. This version is based on [mxlink](https://crates.io/crates/mxlink)@1.4.0 (which is based on the newly released [matrix-sdk](https://crates.io/crates/matrix-sdk)@[0.8.0](https://github.com/matrix-org/matrix-rust-sdk/releases/tag/matrix-sdk-0.8.0)). Once you run this version at least once and your matrix-sdk datastore gets upgraded to the new schema, **you will not be able to downgrade to older baibot versions** (based on the older matrix-sdk), unless you start with an empty datastore.
+
+- (**Bugfix**) Add missing typing notices sending functionality while generating images ([9d166e35ba](https://github.com/etkecc/baibot/commit/9d166e35ba6fc0daaf69318870e92436f3302056))
+
+- (**Feature**) Support for [Matrix authenticated media](https://matrix.org/docs/spec-guides/authed-media-servers/), thanks to upgrading [mxlink](https://crates.io/crates/mxlink) / [matrix-sdk](https://crates.io/crates/matrix-sdk) - fixes [issue #12](https://github.com/etkecc/baibot/issues/12)
+
+
+# (2024-11-12) Version 1.3.2
+
+Dependency updates.
+
+
+# (2024-10-03) Version 1.3.1
+
+- (**Improvement**) Improves fallback user mentions support for old clients (like Element iOS) which use the bot's display name (not its full Matrix User ID). ([d9a045a5e4](https://github.com/etkecc/baibot/commit/d9a045a5e41d2b99694f92ec9e90f47529546d89))
+
+
+# (2024-10-03) Version 1.3.0
+
+**TLDR**: you can now use OpenAI's [o1](https://platform.openai.com/docs/models/o1) models, benefit from [prompt caching](https://platform.openai.com/docs/guides/prompt-caching) and mention the bot again from old clients lacking proper [user mentions support](https://spec.matrix.org/latest/client-server-api/#user-and-room-mentions) (like Element iOS).
+
+- (**Feature**) Introduces a new `baibot_conversation_start_time_utc` [prompt variable](./docs/configuration/text-generation.md#️-prompt-override) which is not a moving target (like the `baibot_now_utc` variable) and allows [prompt caching](https://platform.openai.com/docs/guides/prompt-caching) to work. All default/sample configs have been adjusted to make use of this new variable, but users need to adjust your existing dynamically-created agents to start using it. ([85e66406dc](https://github.com/etkecc/baibot/commit/85e66406dc6f430741c7819f420e2df4ae6e8d3b))
+
+- (**Improvement**) Allows for the `max_response_tokens` configuration value for the [OpenAI provider](./docs/providers.md#openai) to be set to `null` to allow [o1](https://platform.openai.com/docs/models/o1) models (which do not support `max_response_tokens`) to be used. See the new o1 sample config [here](./docs/sample-provider-configs/openai-o1.yml). ([db9422740c](https://github.com/etkecc/baibot/commit/db9422740ceca32956d9628b6326b8be206344e2))
+
+- (**Improvement**) Switches the sample configs for the [OpenAI provider](./docs/providers.md#openai) to point to the `gpt-4o` model, which since 2024-10-02 is the same as the `gpt-4o-2024-08-06` model. We previously explicitly pointed the bot to the `gpt-4o-2024-08-06` model, because it was much better (longer context window). Now that `gpt-4o` points to the same powerful model, we don't need to pin its version anymore. Existing users may wish to adjust their configuration to match. ([90fbad5b64](https://github.com/etkecc/baibot/commit/90fbad5b643cd06c23179f055a309ec6a7cba161))
+
+- (**Bugfix**) Restores fallback user mentions support (via regular text, not via the [user mentions spec](https://spec.matrix.org/latest/client-server-api/#user-and-room-mentions)) to allow certain old clients (like Element iOS) to be able to mention the bot again. Support for this was intentionally removed recently (in [v1.2.0](#2024-10-01-version-120)), but it turned out to be too early to do this. ([b40226826f](https://github.com/etkecc/baibot/commit/b40226826fe914d0d5d265230ebc5bac8058b6f7))
+
+
+# (2024-10-01) Version 1.2.0
+
+- (**Feature**) Adds support for [on-demand involvement](./docs/features.md#on-demand-involvement) of the bot (via mention) in arbitrary threads and reply chains ([9908512968](https://github.com/etkecc/baibot/commit/990851296828168c2106eb3f4668833e9e5a7463)) - fixes [issue #15](https://github.com/etkecc/baibot/issues/15)
+
+- (**Improvement**) Simplifies [Transcribe-only mode](./docs/features.md#transcribe-only-mode) reply format (removing `> 🦻` prefixing) to allow easier forwarding, etc. ([e6aa956423](https://github.com/etkecc/baibot/commit/e6aa95642376ee7d87932d0e66dcfedf261b188b)) - fixes [issue #14](https://github.com/etkecc/baibot/issues/14)
+
+- (**Bugfix**) Fixes speech-to-text replies rendering incorrectly in certain clients, due to them confusing our old reply format with [fallback for rich replies](https://spec.matrix.org/v1.11/client-server-api/#fallbacks-for-rich-replies) ([e6aa956423](https://github.com/etkecc/baibot/commit/e6aa95642376ee7d87932d0e66dcfedf261b188b)) - fixes [issue #17](https://github.com/etkecc/baibot/issues/17)
+
+
+# (2024-09-22) Version 1.1.1
+
+- (**Bugfix**) Fix thread messages being lost due to lack of pagination support ([d4ddd29660](https://github.com/etkecc/baibot/commit/d4ddd29660d9f51d248119dd6032e68ab29e7d35)) - fixes [issue #13](https://github.com/etkecc/baibot/issues/13)
+
+- (**Bugfix**) Fix Anthropic conversations getting stuck when being impatient and sending multiple consecutive messages ([8b12bdf2b3](https://github.com/etkecc/baibot/commit/8b12bdf2b3196abea0e8db33d7c50fff48341cb9)) - fixes [issue #13](https://github.com/etkecc/baibot/issues/13)
+
+
+# (2024-09-21) Version 1.1.0
+
+- (**Feature**) Adds support for [prompt variables](./docs/configuration/text-generation.md#️-prompt-override) (date/time, bot name, model id) ([2a5a2d6a4d](https://github.com/etkecc/baibot/commit/2a5a2d6a4dbf5fd7cb504ac07d4187fdc32ae395)) - fixes [issue #10](https://github.com/etkecc/baibot/issues/10)
+
+- (**Improvement**) [Dockerfile](./Dockerfile) changes to produce ~20MB smaller container images ([354063abb7](https://github.com/etkecc/baibot/commit/354063abb79035069bd3b26c53214874e9cdd95d))
+
+- (**Improvement**) [Dockerfile](./Dockerfile) changes to optimize local (debug) runs in a container ([c8c5e0e540](https://github.com/etkecc/baibot/commit/c8c5e0e540ab981e849452eb3ddb0378105e1fc6))
+
+- (**Improvement**) CI changes to try and work around multi-arch image issues like [this one](https://github.com/etkecc/baibot/issues/2) ([5de7559ed6](https://github.com/etkecc/baibot/commit/5de7559ed685a41c22dfc12283681f02f4c2ee00))
+
+
+# (2024-09-19) Version 1.0.6
+
+Improvements to:
+
+- messages sent by the bot - better onboarding flow, especially when no agents have been created yet
+- documentation pages
+
+
+# (2024-09-14) Version 1.0.5
+
+Further [improves](https://github.com/etkecc/baibot/commit/3b25b92a81a05ebaf1c6dbabf675fbfbe6c9f418) the typing notification logic, so that it tolerates edge cases better.
+
+
+# (2024-09-14) Version 1.0.4
+
+[Improves](https://github.com/etkecc/baibot/commit/dd1dd78312e3db7f92b37fb3b4750fbe35de7115) the typing notification logic.
+
+
+# (2024-09-13) Version 1.0.3
+
+Contains [fixes](https://github.com/etkecc/rust-mxlink/commit/f339fc85e69aa7f614394ad303d1614cd307319c) for [some](https://github.com/etkecc/baibot/issues/1) startup failures caused by partial initialization (errors during startup).
+
+
+# (2024-09-12) Version 1.0.0
+
+Initial release. 🎉

Cargo.toml

@@ -7,31 +7,34 @@ license = "AGPL-3.0-or-later"
 readme = "README.md"
 keywords = ["matrix", "chat", "bot", "AI", "LLM"]
 include = ["/etc/assets/baibot-torso-768.png", "/src", "/README.md", "/CHANGELOG.md", "/LICENSE"]
-version = "1.0.3"
-edition = "2021"
+version = "1.14.3"
+edition = "2024"
 
 [lib]
 name = "baibot"
 path = "src/lib.rs"
 
 [dependencies]
-anthropic-rs = "0.1.*"
+anthropic = { git = "https://github.com/etkecc/anthropic-rs.git", branch = "fix-content-block-image" }
 anyhow = "1.0.*"
-async-openai = "0.24.*"
+async-openai = { version = "0.33.0", features = ["audio", "chat-completion", "image", "responses"] }
 base64 = "0.22.*"
+chrono = { version = "0.4.*", default-features = false, features = ["std", "now"] }
 # We'd rather not depend on this, but we cannot use the ruma-events EventContent macro without it.
-matrix-sdk = { version = "0.7.1", default-features = false }
+# We add the `native-tls` feature, because of https://github.com/etkecc/rust-mxlink/issues/1
+matrix-sdk = { version = "0.16.0", default-features = false, features = ["native-tls"] }
+mime_guess = "2.0.*"
 mxidwc = "1.0.*"
-mxlink = "1.1.*"
+mxlink = ">=1.13.0"
 etke_openai_api_rust = "0.1.*"
 quick_cache = "0.6.*"
-regex = "1.10.*"
+regex = "1.12.*"
 serde = { version = "1.0.*", features = ["derive"], default-features = false }
 serde_json = "1.0.*"
-serde_yaml = "0.9.*"
-tempfile = "3.12.*"
-tiktoken-rs = { version = "0.5.*", features = ["async-openai"] }
-tokio = { version = "1.40.*", features = ["rt", "rt-multi-thread", "macros"] }
+serde_yaml_ng = "0.10.*"
+tempfile = "3.26.*"
+tiktoken-rs = { version = "0.9.*", default-features = false }
+tokio = { version = "1.50.*", features = ["rt", "rt-multi-thread", "macros"] }
 tracing = "0.1.*"
 tracing-subscriber = { version = "0.3.*", features = ["env-filter"] }
 url = "2.5.*"

Dockerfile

@@ -4,7 +4,7 @@
 #                                     #
 #######################################
 
-FROM docker.io/rust:1.80.1-slim-bookworm AS build
+FROM docker.io/rust:1.93.1-slim-trixie AS build
 
 RUN apt-get update && apt-get install -y build-essential pkg-config libssl-dev libsqlite3-dev
 
@@ -15,14 +15,23 @@ WORKDIR /app
 
 COPY . /app
 
+ARG RELEASE_BUILD=true
+
 RUN --mount=type=cache,target=/cargo,sharing=locked \
 	--mount=type=cache,target=/target,sharing=locked \
-	cargo build --release
+	if [ "$RELEASE_BUILD" = "true" ]; then \
+		cargo build --release; \
+	else \
+		cargo build; \
+	fi
 
 # Move it out of the mounted cache, so we can copy it in the next stage.
 RUN --mount=type=cache,target=/target,sharing=locked \
-	cp /target/release/baibot /baibot
-
+	if [ "$RELEASE_BUILD" = "true" ]; then \
+		cp /target/release/baibot /baibot; \
+	else \
+		cp /target/debug/baibot /baibot; \
+	fi
 
 #######################################
 #                                     #
@@ -30,9 +39,11 @@ RUN --mount=type=cache,target=/target,sharing=locked \
 #                                     #
 #######################################
 
-FROM docker.io/debian:bookworm-slim
+FROM docker.io/debian:trixie-slim
 
-RUN apt-get update && apt-get install -y ca-certificates sqlite3
+RUN apt-get update && apt-get install -y ca-certificates sqlite3 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 

Dockerfile.ci

@@ -0,0 +1,35 @@
+#######################################
+#                                     #
+# Stage 1: building                   #
+#                                     #
+#######################################
+
+FROM docker.io/rust:1.93.1-slim-trixie AS build
+
+RUN apt-get update && apt-get install -y build-essential pkg-config libssl-dev libsqlite3-dev
+
+WORKDIR /app
+
+COPY . /app
+
+RUN cargo build --release
+
+#######################################
+#                                     #
+# Stage 2: packaging                  #
+#                                     #
+#######################################
+
+FROM docker.io/debian:trixie-slim
+
+RUN apt-get update && apt-get install -y ca-certificates sqlite3 && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY --from=build /app/target/release/baibot .
+
+ENTRYPOINT ["/bin/sh", "-c"]
+
+CMD ["/app/baibot"]

README.md

@@ -17,10 +17,10 @@ It's influenced by [chaz](https://github.com/arcuru/chaz), but does **not** use
 
 - Supports **different use purposes** (depending on the [☁️ provider](./docs/providers.md) & model):
 
-  - [💬 text-generation](./docs/features.md#-text-generation): communicating with you via text
+  - [💬 text-generation](./docs/features.md#-text-generation): communicating with you via text (though certain models may "see" images as well). The [OpenAI provider](./docs/providers.md#openai) also supports [🛠️ built-in tools](./docs/features.md#️-built-in-tools-openai-only) (web search, code interpreter)
   - [🦻 speech-to-text](./docs/features.md#-speech-to-text): turning your voice messages into text
   - [🗣️ text-to-speech](./docs/features.md#%EF%B8%8F-text-to-speech): turning bot or users text messages into voice messages
-  - [🖌️ image-generation](./docs/features.md#%EF%B8%8F-image-generation): generating images based on instructions
+  - [🖌️ image-generation](./docs/features.md#image-generation): creating and editing images based on instructions
 
 - 🪄 Supports [seamless voice interaction](./docs/features.md#seamless-voice-interaction) (turning user voice messages into text, answering in text, then turning that text back into voice)
 
@@ -41,7 +41,7 @@ It's influenced by [chaz](https://github.com/arcuru/chaz), but does **not** use
 
 ![Introduction and general usage](./docs/screenshots/introduction-and-general-usage.webp)
 
-You can find more screenshots on the the [🌟 Features](./docs/features.md) and other [📚 Documentation](./docs/README.md) pages, as well as in the [docs/screenshots](./docs/screenshots) directory.
+You can find more screenshots on the [🌟 Features](./docs/features.md) and other [📚 Documentation](./docs/README.md) pages, as well as in the [docs/screenshots](./docs/screenshots) directory.
 
 
 ## 🚀 Getting Started

docs/access.md

@@ -5,17 +5,22 @@ This bot employs access control to decide who can use its services and manage it
 
 ### 👋 Joining rooms
 
-The bot automatically joins rooms when invited by someone considered a bot [user](#-users).
+The bot automatically joins rooms only when invited by someone considered a bot [👥 user](#-users).
 
 
 ### 👥 Users
 
-The bot will ignore messages (and room invitations) from unallowed users.
-
-Users can **use all the bot's [features](./features.md)** ([💬 Text Generation](./features.md#-text-generation), [🦻 Speech-to-Text](./features.md#-speech-to-text), etc.), but **cannot manage the bot's configuration**.
-
 The bot can be used by users that match some [dynamically](./configuration/README.md#dynamic-configuration) configured [Matrix user id](https://spec.matrix.org/v1.11/#users) patterns.
 
+Users:
+
+- ✅ can **invite the bot to rooms**
+- ✅ can **use all the bot's [features](./features.md)** ([💬 Text Generation](./features.md#-text-generation), [🦻 Speech-to-Text](./features.md#-speech-to-text), etc.) by sending room messages
+- ✅ can **mention the bot** in threads and reply chains to provoke it to respond to non-user messages (see [🌟 Features / 💬 Text Generation / On-demand involvement](./features.md#on-demand-involvement))
+- ✅ can **change the bot's configuration in a room** (e.g. `!bai config room ...` commands)
+- ❌ cannot **change the bot's global configuration** (e.g. `!bai config global ...` commands)
+- ❌ cannot **create new [🤖 Agents](./agents.md)** (neither in rooms, nor globally). See [💼 Room-local agent managers](#-room-local-agent-managers) for controlling which users can create agents.
+
 The following commands are available:
 - **Show** the currently allowed users: `!bai access users`
 - **Set** the list of allowed users: `!bai access set-users SPACE_SEPARATED_PATTERNS`
@@ -27,6 +32,8 @@ Example patterns: `@*:example.com @*:another.com @someone:company.org`
 
 Administrators can **manage the bot's configuration and access control**.
 
+Administrators are [👥 Users](#-users) and [💼 Room-local agent managers](#-room-local-agent-managers) implicitly, so they inherit all their permissions.
+
 The bot can be administrated by users that match some [statically](./configuration/README.md#static-configuration) configured [Matrix user id](https://spec.matrix.org/v1.11/#users) patterns.
 
 Administrators cannot be changed without adjusting the bot's configuration on the server.
@@ -35,12 +42,12 @@ Administrators cannot be changed without adjusting the bot's configuration on th
 ### 💼 Room-local agent managers
 
 Room-local agent managers are users privileged to **create their own [agents](./agents.md)** (see `!bai agent`) in rooms.
-Letting regular users create agents which contact arbitrary network services **may be a security issue**.
 
-No room-local agent manager patterns are configured, so new agents can only be created by administrators.
+> [!WARNING]
+> Letting regular users create agents which contact arbitrary network services **may be a security issue**.
 
 The following commands are available:
 - **Show** the currently allowed users: `!bai access room-local-agent-managers`
 - **Set** the list of allowed users: `!bai access set-room-local-agent-managers SPACE_SEPARATED_PATTERNS`
 
-Example patterns: `@*:synapse.127.0.0.1.nip.io @*:another.com @someone:company.org`
+Example patterns: `@*:example.com @*:another.com @someone:company.org`

docs/agents.md

@@ -35,7 +35,7 @@ Depending on where the agent is defined (within a room, globally, or [statically
 
 When creating an agent, you will be given some sample [YAML](https://en.wikipedia.org/wiki/YAML) configuration which you can use to customize the agent's behavior.
 
-This configuration varies depending on the [☁️ provider](./providers.md) used and the capabilities of the agent. Based on the configuration keys you pass, certain features will be enabled or disabled. For example, if you skip the `image_generation` key for an [OpenAI](./providers.md#openai) agent, it won't be able to generate images (see [🖌️ Image Generation](./features.md#-image-generation)).
+This configuration varies depending on the [☁️ provider](./providers.md) used and the capabilities of the agent. Based on the configuration keys you pass, certain features will be enabled or disabled. For example, if you skip the `image_generation` key for an [OpenAI](./providers.md#openai) agent, it won't be able to generate images (see [🖌️ Image Creation](./features.md#-image-creation), [🎨 Image Editing](./features.md#-image-editing), [🫵 Sticker Creation](./features.md#-sticker-creation)).
 
 After making your modifications to the sample YAML, you submit it back to the bot and the new agent will be created.
 

docs/configuration/README.md

@@ -12,12 +12,15 @@ This file is created from the template found in [etc/app/config.yml.dist](../../
 
 Certain keys can be left unset, in which case [📝 hardcoded defaults](../../src/entity/cfg/defaults.rs) would be used.
 
-Each configuration key found in the YAML configuration can be overridden by setting an environment variable (dots should be replaced with `_`). Example:
+Some configuration keys found in the YAML configuration can be overridden by setting an environment variable (dots should be replaced with `_`). Example:
 
 - to override `command_prefix`, set an environment variable `BAIBOT_COMMAND_PREFIX`
 - to override `homeserver.server_name`, set an environment variable `BAIBOT_HOMESERVER_SERVER_NAME`
 
-The static configuration contains an `initial_global_config` key, which is used to populate the bot's global configuration (stored as [dynamic configuration](#dynamic-configuration)) the first time the bot starts. Modifying this subsequently will not have any effect. After initial global configuration creation, it's expected to be managed dynamically via chat commands.
+You can see the list of supported environment variables in the [🦀 src/entity/cfg/env.rs](../../src/entity/cfg/env.rs) file.
+
+> [!WARNING]
+> The static configuration contains an `initial_global_config` key, which is used to populate the bot's global configuration (stored as [dynamic configuration](#dynamic-configuration)) the first time the bot starts. Modifying this subsequently will not have any effect. After initial global configuration creation, it's expected to be managed dynamically via chat commands.
 
 
 ### Dynamic configuration
@@ -40,7 +43,7 @@ You can adjust the following settings per room and/or globally:
 - [💬 Text Generation](text-generation.md)
 - [🦻 Speech-to-Text](speech-to-text.md)
 - [🗣️ Text-to-Speech](text-to-speech.md)
-- [🖌️ Image Generation](image-generation.md)
+- [🖌️ Image Creation](image-generation.md)
 - [🤝 Handlers](handlers.md)
 
 Refer to the bot's help messages (as a response to a `!bai config` help command) for the most up-to-date information on what Room Settings can be configured.

docs/configuration/handlers.md

@@ -8,10 +8,10 @@ You can also use **different models within the same room** (e.g. [💬 text-gene
 
 The bot supports the following use-purposes:
 
-- [💬 text-generation](../features.md#-text-generation): communicating with you via text
+- [💬 text-generation](../features.md#-text-generation): communicating with you via text (though certain models may "see" images as well)
 - [🦻 speech-to-text](../features.md#-speech-to-text): turning your voice messages into text
 - [🗣️ text-to-speech](../features.md#️-text-to-speech): turning bot or users text messages into voice messages
-- [🖌️ image-generation](../features.md#-image-generation): generating images based on instructions
+- [🖌️ image-generation](../features.md#image-generation): generating images based on instructions
 
 In a given room, each different purpose can be served by a different [provider](../providers.md) and model. This combination of provider and model configuration is called an [🤖 agent](../agents.md). Each purpose can be served by a different **handler** agent.
 

docs/configuration/image-generation.md

@@ -1,9 +1,11 @@
 
-## 🖌️ Image Generation
+## Image Generation
 
-The Image Generation feature is not configurable at this moment.
+The Image Creation and Image Editing features are not configurable at this moment.
 
 You may also wish to see:
 
-- [🌟 Features / 🖌️ Image Generation](../features.md#-image-generation) for a higher-level introduction to the Image Generation features
-- [📖 Usage / 🖌️ Image Generation](../usage.md#-image-generation) section for more details on how to use the bot for Image Generation in a room
+- [🌟 Features / Image Generation / 🖌️ Image Creation](../features.md#-image-creation) for a higher-level introduction to the Image Creation features
+- [🌟 Features / Image Generation / 🎨 Image Editing](../features.md#-image-editing) for a higher-level introduction to the Image Editing features
+- [📖 Usage / Image Generation / 🖌️ Creating Images](../usage.md#-creating-images) section for more details on how to use the bot for Image Creation in a room
+- [📖 Usage / Image Generation / 🎨 Editing images](../usage.md#-editing-images) section for more details on how to use the bot for Image Editing in a room

docs/configuration/speech-to-text.md

@@ -23,6 +23,19 @@ The following configuration values are recognized:
 Example: `!bai config room speech-to-text set-flow-type ignore` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings))
 
 
+### 🪄 Message Type for non-threaded only-transcribed messages
+
+Controls how the transcribed text of voice messages is sent to the chat when Flow Type = `only_transcribe`.
+
+The following configuration values are recognized:
+
+- (default) `text`: the transcribed text is sent as a regular message. This is more convenient if you'd like to forward the transcribed message to other rooms.
+
+- `notice`: the transcribed text is sent as a notice message. This provides better compatibility with other bots in the room, as they are less likely to interact with messages of type notice.
+
+Example: `!bai config room speech-to-text set-msg-type-for-non-threaded-only-transcribed-messages notice` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings))
+
+
 ### 🔤 Language
 
 Lets you specify the language of the input voice messages, to avoid using auto-detection.

docs/configuration/text-generation.md

@@ -13,7 +13,7 @@ You may also wish to see:
 
 In Direct Message rooms with the bot (1:1 rooms), it most usually makes sense for the bot to respond to **all** of your messages, as shown on this [🖼️ screenshot](../screenshots/text-generation.webp).
 
-In group rooms (with multiple users), it may be more appropriate for the bot to only respond to messages that are **prefixed** with the command prefix (e.g. `!bai`), so that other chat exchange in the room will not trigger it. Such a setup is shown on this [🖼️ screenshot](../screenshots/text-generation-prefix-requirement.webp).
+In group rooms (with multiple users), it may be more appropriate for the bot to only respond to messages that are **prefixed** with the command prefix (e.g. `!bai`) or which are [mentioning](https://spec.matrix.org/latest/client-server-api/#user-and-room-mentions) the bot (e.g. `@baibot`), so that other chat exchange in the room will not trigger it. Such a setup is shown on the [🖼️ On-demand involvement in the room](../screenshots/text-generation-prefix-requirement.webp) screenshot.
 
 There are exceptions to these rules, and you can configure the bot to respond only to prefixed messages in a 1:1 room, or to respond to all messages even in a multi-user group room.
 
@@ -27,7 +27,10 @@ By default, the bot is **auto-configured (upon joining a new room)** to use the
 
 Example: `!bai config room text-generation set-prefix-requirement-type command_prefix` (this can also be set globally, see [🛠️ Room Settings](./README.md#room-settings))
 
-Regardless of this configuration, **the bot will also respond to messages which directly [mention](https://spec.matrix.org/latest/client-server-api/#user-and-room-mentions) the bot** (e.g. `@baibot`), even if they are not prefixed. An example of this can be seen on this [🖼️ screenshot](../screenshots/text-generation-prefix-requirement.webp).
+Regardless of this configuration, **the bot will also respond to messages by allowed [👥 Users](../access.md#-users) which directly [mention](https://spec.matrix.org/latest/client-server-api/#user-and-room-mentions) the bot** (e.g. `@baibot`), even if they are not prefixed. An example of this can be seen on these screenshots:
+
+- [🖼️ On-demand involvement in a thread](../screenshots/text-generation-on-demand-thread-involvement.webp)
+- [🖼️ On-demand involvement in a reply chain](../screenshots/text-generation-on-demand-reply-involvement.webp)
 
 
 ### 🪄 Auto Usage
@@ -68,6 +71,22 @@ Where appropriate, you'll mention best practices and common pitfalls.
 
 A prompt override can also be set globally, see [🛠️ Room Settings](./README.md#room-settings).
 
+Prompts may contain the following **placeholder variables** which will be replaced *every time* the bot is interacted with:
+
+| Placeholder               | Description | Example |
+|---------------------------|-------------|---------|
+| `{{ baibot_name }}`       | Name of the bot as configured in the `user.name` field in the [Static configuration](./README.md#static-configuration) | `Baibot` |
+| `{{ baibot_model_id }}`   | Text-Generation model ID as configured in the [🤖 agent](../agents.md)'s configuration | `gpt-4o` |
+| `{{ baibot_now_utc }}`    | Current date and time in UTC (⚠️ usage may break prompt caching - see below) | `2024-09-20 (Friday), 14:26:42 UTC` |
+| `{{ baibot_conversation_start_time_utc }}` | The date and time in UTC that the conversation started | `2024-09-20 (Friday), 14:26:42 UTC` |
+
+💡 `{{ baibot_now_utc }}` changes as time goes on, which prevents [prompt caching](https://platform.openai.com/docs/guides/prompt-caching) from working. It's better to use `{{ baibot_conversation_start_time_utc }}` in prompts, as its value doesn't change yet still orients the bot to the current date/time.
+
+Here's a prompt that combines some of the above variables:
+
+> You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
+
+
 ### 🌡️ Temperature Override
 
 You can override the [temperature](https://blogs.novita.ai/what-are-large-language-model-settings-temperature-top-p-and-max-tokens/#what-is-llm-temperature) (randomness / creativity) parameter configured at the [🤖 agent](../agents.md) level.

docs/development.md

@@ -18,6 +18,27 @@ For local development, we run all dependency services in [🐋 Docker](https://w
 - (Optional) an API key for some Large Language Model [☁️ provider](./providers.md) (e.g. [OpenAI](./providers.md#openai)), though we recommend using [LocalAI](#localai) or [Ollama](#ollama) for local development
 
 
+### Choosing a homeserver
+
+The development environment supports two homeserver implementations:
+
+- **[Continuwuity](https://continuwuity.org/)** (default) — lightweight, no external database required. Good for most development needs.
+- **[Synapse](https://github.com/element-hq/synapse)** — the reference implementation, bundled with Postgres. Use this if you need Synapse-specific behavior.
+
+To choose a homeserver (optional — defaults to Continuwuity if skipped):
+
+```sh
+just homeserver-init continuwuity  # or: just homeserver-init synapse
+```
+
+The choice is stored in `var/homeserver` and affects all subsequent commands.
+
+> **Note:** If you switch homeservers after initial setup, you will need to:
+> - Delete `var/app/local/` and/or `var/app/container/` (app config and data)
+> - Delete `var/services/element-web/` (to regenerate its config)
+> - Re-run the prepare and user registration steps
+
+
 ### Getting started guide
 
 Developing [locally](#running-locally) is possible, but requires a [Rust](https://www.rust-lang.org/) toolchain.
@@ -28,11 +49,12 @@ In any case, you will need [🐋 Docker](https://www.docker.com/) as [dependency
 
 #### Running locally
 
-1. Start the core dependency services (Postgres, Synapse, Element Web): `just services-start`
-2. (Only the first time around) Prepare initial app configuration in `var/app/local/config.yml`: `just app-local-prepare`
-3. (Only the first time around) [Prepare your configuration file](#prepare-your-configuration-file)
-4. (Only the first time around) Prepare initial default Matrix user accounts (`admin` and `baibot`): `just users-prepare`
-5. (Optional) Start additional services depending on which [agent provider you've chosen](#choosing-an-agent-provider):
+1. (Optional) Choose a homeserver: `just homeserver-init continuwuity` (or `synapse`). Default is `continuwuity`.
+2. Start the homeserver and Element Web: `just services-start`
+3. (Only the first time around) Prepare initial app configuration in `var/app/local/config.yml`: `just app-local-prepare`
+4. (Only the first time around) [Prepare your configuration file](#prepare-your-configuration-file)
+5. (Only the first time around) Prepare initial default Matrix user accounts (`admin` and `baibot`): `just users-prepare`
+6. (Optional) Start additional services depending on which [agent provider you've chosen](#choosing-an-agent-provider):
   - for [LocalAI](#localai):
     - Start services: `just localai-start`
 	- Wait a while for LocalAI to start up. It has a lot of models to download. Monitor progress using `just localai-tail-logs`
@@ -40,12 +62,12 @@ In any case, you will need [🐋 Docker](https://www.docker.com/) as [dependency
   - for [Ollama](#ollama):
     - Start services: `just ollama-start`
     - (Only the first time around) Pull the model configured in `agents.static_definitions` in the configuration file: `just ollama-pull-model gemma2:2b`
-6. Start the bot: `just run-locally`
-7. Go to http://element.127.0.0.1.nip.io:42025/ and login with `admin` / `admin`
-8. Create a new room and invite `@baibot:synapse.127.0.0.1.nip.io`
-9. When done, stop the bot (`Ctrl` + `C`)
-10. Stop the core dependency services: `just services-stop`
-11. (Optional) Stop additional services:
+7. Start the bot: `just run-locally`
+8. Go to http://element.127.0.0.1.nip.io:42025/ and login with `admin` / `admin`
+9. Create a new room and invite `@baibot:continuwuity.127.0.0.1.nip.io` (or `@baibot:synapse.127.0.0.1.nip.io` if using Synapse)
+10. When done, stop the bot (`Ctrl` + `C`)
+11. Stop the services: `just services-stop`
+12. (Optional) Stop additional services:
   - for [LocalAI](#localai): `just localai-stop`
   - for [Ollama](#ollama): `just ollama-stop`
 
@@ -54,11 +76,12 @@ In any case, you will need [🐋 Docker](https://www.docker.com/) as [dependency
 
 You can avoid having a [Rust](https://www.rust-lang.org/) toolchain installed locally and build/run this in a container.
 
-1. Start the core dependency services (Postgres, Synapse, Element Web): `just services-start`
-2. (Only the first time around) Prepare initial app configuration in `var/app/container/config.yml`: `just app-container-prepare`
-3. (Only the first time around) [Prepare your configuration file](#prepare-your-configuration-file)
-4. (Only the first time around) Prepare initial default Matrix user accounts (`admin` and `baibot`): `just users-prepare`
-5. (Optional) Start additional services depending on which [agent provider you've chosen](#choosing-an-agent-provider):
+1. (Optional) Choose a homeserver: `just homeserver-init continuwuity` (or `synapse`). Default is `continuwuity`.
+2. Start the homeserver and Element Web: `just services-start`
+3. (Only the first time around) Prepare initial app configuration in `var/app/container/config.yml`: `just app-container-prepare`
+4. (Only the first time around) [Prepare your configuration file](#prepare-your-configuration-file)
+5. (Only the first time around) Prepare initial default Matrix user accounts (`admin` and `baibot`): `just users-prepare`
+6. (Optional) Start additional services depending on which [agent provider you've chosen](#choosing-an-agent-provider):
   - for [LocalAI](#localai):
     - Start services: `just localai-start`
 	- Wait a while for LocalAI to start up. It has a lot of models to download. Monitor progress using `just localai-tail-logs`
@@ -66,12 +89,12 @@ You can avoid having a [Rust](https://www.rust-lang.org/) toolchain installed lo
   - for [Ollama](#ollama):
     - Start services: `just ollama-start`
     - (Only the first time around) Pull the model configured in `agents.static_definitions` in the configuration file: `just ollama-pull-model gemma2:2b`
-6. Start the bot: `just run-in-container`
-7. Go to http://element.127.0.0.1.nip.io:42025/ and login with `admin` / `admin`
-8. Create a new room and invite `@baibot:synapse.127.0.0.1.nip.io`
-9. When done, stop the bot (`Ctrl` + `C`)
-10. Stop the dependency services: `just services-stop`
-11. (Optional) Stop additional services:
+7. Start the bot: `just run-in-container`
+8. Go to http://element.127.0.0.1.nip.io:42025/ and login with `admin` / `admin`
+9. Create a new room and invite `@baibot:continuwuity.127.0.0.1.nip.io` (or `@baibot:synapse.127.0.0.1.nip.io` if using Synapse)
+10. When done, stop the bot (`Ctrl` + `C`)
+11. Stop the services: `just services-stop`
+12. (Optional) Stop additional services:
   - for [LocalAI](#localai): `just localai-stop`
   - for [Ollama](#ollama): `just ollama-stop`
 
@@ -93,7 +116,7 @@ For getting started most quickly (and locally), we recommend using [LocalAI](#lo
 
 **Ollama is most lightweight** (~2GB for the container image + ~1.6GB for the model), but supports only [💬 text-generation](./features.md#-text-generation).
 
-**LocalAI requires 4x more disk space** (~6GB for the container image + ~12GB for the models), but supports [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text) and [🖼️ image-generation](./features.md#️-image-generation).
+**LocalAI requires 4x more disk space** (~6GB for the container image + ~12GB for the models), but supports [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text) and [🖼️ image-generation](./features.md#️-image-creation).
 
 **OpenAI supports all of these capabilities** as well and does not require powerful hardware or lots of disk space. However, it requires signup and an API key.
 

docs/features.md

@@ -8,7 +8,7 @@ You can also use **different models within the same room** (e.g. [💬 text-gene
 
 The bot supports the following use-purposes:
 
-- [💬 text-generation](#-text-generation): communicating with you via text
+- [💬 text-generation](#-text-generation): communicating with you via text (though certain models may "see" images as well)
 - [🦻 speech-to-text](#-speech-to-text): turning your voice messages into text
 - [🗣️ text-to-speech](#%EF%B8%8F-text-to-speech): turning bot or users text messages into voice messages
 - [🖌️ image-generation](#%EF%B8%8F-image-generation): generating images based on instructions
@@ -22,12 +22,16 @@ For more information about configuring handlers, see the [🤝 Handlers / Config
 
 ### 💬 Text Generation
 
-Text Generation is the bot's ability to **respond to users' text messages with text**.
+Text Generation is the bot's ability to **respond to users' messages with text**.
 
 ![Screenshot of Text Generation - a user sends a message and the bot replies in a new conversation thread](./screenshots/text-generation.webp)
 
+Some models also support vision, so you may be able to mix text and images in the same conversation.
+
 In multi-user (group) rooms, to avoid disturbing the normal conversation between people, the bot is auto-configured to only respond to messages starting with the command prefix (`!bai`) or direct mentions via the [💬 Text Generation / 🗟 Prefix Requirement Type](./configuration/text-generation.md#-prefix-requirement-type) setting.
 
+Normally, the bot only responds to allowed [👥 Users](./access.md#-users). In certain cases, it's useful for an allowed user to provoke the bot to respond even in foreign threads or reply chains. You can learn more about this feature in the [On-demand involvement](./features.md#on-demand-involvement) section below.
+
 A few other features (like [🗣️ Text-to-Speech](#️-text-to-speech) and [🦻 Speech-to-Text](#-speech-to-text)) combine well with Text Generation, so you **don't necessarily need to communicate with the bot via text** (with [Seamless voice interaction](#seamless-voice-interaction), you can communicate only with voice).
 
 You may also wish to see:
@@ -36,6 +40,39 @@ You may also wish to see:
 - [📖 Usage / 💬 Text Generation](./usage.md#-text-generation) section for more details on how to use the bot for Text Generation in a room
 
 
+#### 🛠️ Built-in Tools (OpenAI only)
+
+
+
+The [OpenAI provider](./providers.md#openai) supports built-in tools that extend the model's capabilities:
+
+- [🔍 Web Search](https://platform.openai.com/docs/guides/tools-web-search) (`web_search`): allows the model to search the web for up-to-date information. [🖼️ Screenshot](./screenshots/text-generation-tools-web-search.webp)
+
+- [💻 Code Interpreter](https://platform.openai.com/docs/guides/tools-code-interpreter) (`code_interpreter`): allows the model to write and execute Python code in a sandbox
+
+These tools are **disabled by default** and need to be explicitly enabled in the agent's `text_generation.tools` configuration. See the [OpenAI sample configuration](https://github.com/etkecc/baibot/blob/c70387b0c38d8d0f30bba2179a2a21a3710dbeaf/docs/sample-provider-configs/openai.yml#L12-L15) for reference.
+
+To enable tools on an existing dynamically-created agent, you need to [update the agent](./agents.md#updating-agents) to re-create it with the `text_generation.tools` section added and enable the tools you need
+
+💡 **Note**: These tools run on OpenAI's infrastructure and may incur additional costs. Web search results include citations that are incorporated into the response.
+
+
+#### On-demand involvement
+
+In the following 2 cases, it's useful to involve the bot in conversations on-demand:
+
+1. In multi-user rooms (with the [🗟 Prefix Requirement](./configuration/text-generation.md#-prefix-requirement-type) setting set to "required")
+2. In rooms with foreign users (users that are not authorized bot [👥 users](./access.md#-users))
+
+In these instances, an allowed [👥 user](./access.md#-users) can also provoke the bot to respond to **any** thread or reply chain by [mentioning](https://spec.matrix.org/latest/client-server-api/#user-and-room-mentions) the bot (e.g. `@baibot Hello!`). The following screenshots demonstrate this behavior:
+
+- [🖼️ On-demand involvement in the room](./screenshots/text-generation-prefix-requirement.webp)
+- [🖼️ On-demand involvement in a thread](./screenshots/text-generation-on-demand-thread-involvement.webp) (the Alice user in this example is not an allowed user, yet her messages are still considered as part of the conversation context)
+- [🖼️ On-demand involvement in a reply chain](./screenshots/text-generation-on-demand-reply-involvement.webp) (the Alice user in this example is not an allowed user, yet her messages are still considered as part of the conversation context)
+
+💡 **NOTE**: Normally, the bot **only considers messages from allowed [👥 Users](./access.md#-users)** and ignores all other messages when responding. However, **when the bot is explicitly invoked (via mention)** in a thread or reply chain, **it will consider all messages** in the thread and reply chain (even those from foreign users) as part of the conversation context.
+
+
 ### 🗣️ Text-to-Speech
 
 Text-to-Speech is the bot's ability to **turn text messages into voice messages**.
@@ -118,27 +155,45 @@ To operate in this mode, you can:
 
 - adjust the [🦻 Speech-to-Text / 🪄 Flow Type](./configuration/speech-to-text.md#-flow-type) setting to make the bot only transcribe (without doing [💬 Text Generation](#-text-generation)): `!bai config room speech-to-text set-flow-type only_transcribe`
 
+- optionally adjust [🦻 Speech-to-Text / 🪄 Message Type for non-threaded only-transcribed messages](./configuration/speech-to-text.md#-message-type-for-non-threaded-only-transcribed-messages), if you'd like to bot to send messages of type `notice` (for better compatibility with other bots in the room) instead of sending regular `text` messages (default)
 
-### 🖌️ Image Generation
 
-Image generation is the bot's ability to **generate images** based on text prompts.
+### Image Generation
 
-See a [🖼️ Screenshot of the Image Generation feature](./screenshots/image-generation.webp).
+#### 🖌️ Image Creation
+
+Image creation is the bot's ability to **create images** based on text prompts.
+
+See a [🖼️ Screenshot of the Image Creation feature](./screenshots/image-creation.webp).
 
 You may also wish to see:
 
 - [🛠️ Configuration / 🖌️ Image Generation](./configuration/image-generation.md) for configuration options related to Image Generation
-- [📖 Usage / 🖌️ Image Generation](./usage.md#-image-generation) section for more details on how to use the bot for Image Generation in a room
-- [🫵 Sticker Generation](#-sticker-generation) - a special case of Image Generation
+- [📖 Usage / Image Generation / 🖌️ Creating Images](./usage.md#-creating-images) section for more details on how to use the bot for Image Creation in a room
+- [🖌️ Image Editing](#️-image-editing) - another image generation feature
+- [🫵 Sticker Creation](#-sticker-creation) - a special case of Image Creation
 
 
-### 🫵 Sticker Generation
+#### 🎨 Image Editing
 
-Sticker generation is the bot's ability to **generate sticker** images based on text prompts. It's a special case of [🖌️ Image Generation](#️-image-generation).
+Image editing is the bot's ability to **edit images** based on a prompt and one or more existing images.
 
-See a [🖼️ Screenshot of the Sticker Generation feature](./screenshots/sticker-generation.webp).
+See a [🖼️ Screenshot of the Image Editing feature (manipulating a single image)](./screenshots/image-editing-single-image.webp) and a [🖼️ Screenshot of the Image Editing feature (manipulating multiple images)](./screenshots/image-editing-multiple-images.webp).
 
-See [📖 Usage / 🖌️ Image Generation / Generating Stickers](./usage.md#generating-stickers) for details.
+You may also wish to see:
+
+- [🛠️ Configuration / 🖌️ Image Generation](./configuration/image-generation.md) for configuration options related to Image Generation
+- [📖 Usage / Image Generation / 🎨 Editing images](./usage.md#-editing-images) section for more details on how to use the bot for Image Editing in a room
+- [🖌️ Image Creation](#️-image-creation) - another image generation feature
+
+
+#### 🫵 Sticker Creation
+
+Sticker generation is the bot's ability to **generate sticker** images based on text prompts. It's a special case of [🖌️ Image Creation](#️-image-creation).
+
+See a [🖼️ Screenshot of the Sticker Creation feature](./screenshots/sticker-generation.webp).
+
+See [📖 Usage / Image Generation / 🫵 Creating Stickers](./usage.md#-creating-stickers) for details.
 
 
 ### 🔒 Encryption

docs/installation.md

@@ -15,8 +15,16 @@
 
 We provide prebuilt container images for the `amd64` and `arm64` architectures, so **you don't necessarily need to build images yourself** and can jump to [Running in a container](#-running-in-a-container).
 
-If you nevertheless wish to build a container image yourself, you can do so by running `just build-container-image`.
-This will build and tag your container image as `localhost/baibot:latest`.
+If you nevertheless wish to build a container image yourself, you can do so by running:
+
+- (recommended) `just build-container-image-release` to build a release version of the container image
+
+- or `just build-container-image-debug` to build a debug version of the container image
+
+Debug images are faster to build but are larger in size.
+Release images are ~5x smaller in size, but are slower to build.
+
+Both of these commands will build and tag your container image as `localhost/baibot:latest`.
 
 
 ### 🐋 Running in a container
@@ -45,6 +53,7 @@ CONTAINER_IMAGE_NAME=ghcr.io/etkecc/baibot:v1.0.0
   --env BAIBOT_PERSISTENCE_DATA_DIR_PATH=/data \
   --mount type=bind,src=/path/to/config.yml,dst=/app/config.yml,ro \
   --mount type=bind,src=/path/to/data,dst=/data \
+  --tmpfs=/tmp:rw,noexec,nosuid,size=1024m \
   $CONTAINER_IMAGE_NAME
 ```
 

docs/providers.md

@@ -23,17 +23,20 @@ The list of supported providers is below.
 
 ### How to choose a provider
 
-If you're not sure which provider to start with, we **recommend [OpenAI](#openai)** as it's the most popular and has the **widest range of capabilities**: [💬 text-generation](./features.md#-text-generation), [🖌️ image-generation](./features.md#️-image-generation), [🦻 speech-to-text](./features.md#-speech-to-text), [🗣️ text-to-speech](./features.md#️-text-to-speech).
+If you're not sure which provider to start with, **we recommend [OpenAI](#openai)** as it's the most popular and has the **widest range of capabilities**: [💬 text-generation](./features.md#-text-generation) (incl. vision, incl. [🛠️ tools](./features.md#️-built-in-tools-openai-only)), [🖌️ image-generation](./features.md#️image-generation), [🦻 speech-to-text](./features.md#-speech-to-text), [🗣️ text-to-speech](./features.md#️-text-to-speech).
 
 You don't need to choose just one though. The bot supports [mixing & matching models](./features.md#-mixing--matching-models), so you can use multiple providers at the same time.
 
 
 ### How to use a provider
 
-- sign up for it
-- obtain an API key
-- [create a new agent](./agents.md#creating-agents)
-- set it as a handler for some types of messages (see [Mixing & matching models](./features.md#-mixing--matching-models)) for a specific room or globally
+1. 📝 **Sign up for it**
+
+2. 🔑 **Obtain an API key**
+
+3. 🤖 **Create one or more agents** in a given room or globally. Next to each provider in the [list below](#supported-providers) you'll see **🗲 Quick start** commands, but you may also refer to the [agent creation guide](./agents.md#creating-agents).
+
+4. 🤝 **Set the new agent as a handler** for a given use-purpose like text-generation, image-generation, etc. The agent creation wizard will tell you how, but you may also refer to the [🤝 Handlers](./configuration/handlers.md) guide.
 
 
 ### Supported providers
@@ -44,7 +47,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
 
 - 🆔 Identifier: `anthropic`
 - 🔗 Links: [🏠 Home page](https://www.anthropic.com/), [🌐 Wiki](https://en.wikipedia.org/wiki/Anthropic), [👤 Sign up](https://console.anthropic.com/), [📋 Models list](https://docs.anthropic.com/en/docs/about-claude/models)
-- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation)
+- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (incl. vision, no tools)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local anthropic my-anthropic-agent`
   - create a global agent: `!bai agent create-global anthropic my-anthropic-agent`
@@ -58,7 +61,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
 
 - 🆔 Identifier: `groq`
 - 🔗 Links: [🏠 Home page](https://groq.com/), [🌐 Wiki](https://en.wikipedia.org/wiki/Groq), [👤 Sign up](https://console.groq.com/login), [📋 Models list](https://console.groq.com/docs/models)
-- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation), [🦻 speech-to-text](./features.md#-speech-to-text)
+- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools), [🦻 speech-to-text](./features.md#-speech-to-text)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local groq my-groq-agent`
   - create a global agent: `!bai agent create-global groq my-groq-agent`
@@ -72,7 +75,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
 
 - 🆔 Identifier: `localai`
 - 🔗 Links: [🏠 Home page](https://localai.io/), [📋 Models list](https://localai.io/gallery.html)
-- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
+- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local localai my-localai-agent`
   - create a global agent: `!bai agent create-global localai my-localai-agent`
@@ -86,7 +89,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
 
 - 🆔 Identifier: `mistral`
 - 🔗 Links: [🏠 Home page](https://mistral.ai/), [🌐 Wiki](https://en.wikipedia.org/wiki/Mistral_AI), [👤 Sign up](https://auth.mistral.ai/ui/registration), [📋 Models list](https://docs.mistral.ai/getting-started/models/)
-- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation)
+- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local mistral my-mistral-agent`
   - create a global agent: `!bai agent create-global mistral my-mistral-agent`
@@ -100,7 +103,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
 
 - 🆔 Identifier: `ollama`
 - 🔗 Links: [🏠 Home page](https://ollama.com/), [📋 Models list](https://ollama.com/library)
-- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation)
+- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local ollama my-ollama-agent`
   - create a global agent: `!bai agent create-global ollama my-ollama-agent`
@@ -117,7 +120,7 @@ For services which are not fully compatible with the OpenAI API, consider using
 
 - 🆔 Identifier: `openai`
 - 🔗 Links: [🏠 Home page](https://openai.com/), [🌐 Wiki](https://en.wikipedia.org/wiki/OpenAI), [👤 Sign up](https://platform.openai.com/signup), [📋 Models list](https://platform.openai.com/docs/models)
-- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-generation), [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
+- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-creation), [💬 text-generation](./features.md#-text-generation) (incl. vision, incl. [🛠️ tools](./features.md#️-built-in-tools-openai-only)), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local openai my-openai-agent`
   - create a global agent: `!bai agent create-global openai my-openai-agent`
@@ -134,7 +137,7 @@ Some of these popular services already have **shortcut** providers (leading to t
 This provider is just as featureful as the [OpenAI](#openai) provider, but is more compatible with services which do not fully adhere to the [OpenAI API spec](https://github.com/openai/openai-openapi/).
 
 - 🆔 Identifier: `openai-compatible`
-- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-generation), [💬 text-generation](./features.md#-text-generation), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
+- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-creation), [💬 text-generation](./features.md#-text-generation) (no vision, no tools), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local openai-compatible my-openai-compatible-agent`
   - create a global agent: `!bai agent create-global openai-compatible my-openai-compatible-agent`
@@ -148,7 +151,7 @@ This provider is just as featureful as the [OpenAI](#openai) provider, but is mo
 
 - 🆔 Identifier: `openrouter`
 - 🔗 Links: [🏠 Home page](https://openrouter.ai/), [👤 Sign up](https://openrouter.ai/), [📋 Models list](https://openrouter.ai/models)
-- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation)
+- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local openrouter my-openrouter-agent`
   - create a global agent: `!bai agent create-global openrouter my-openrouter-agent`
@@ -162,7 +165,7 @@ This provider is just as featureful as the [OpenAI](#openai) provider, but is mo
 
 - 🆔 Identifier: `together-ai`
 - 🔗 Links: [🏠 Home page](https://www.together.ai/), [👤 Sign up](https://api.together.ai/signup), [📋 Models list](https://api.together.xyz/models)
-- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation)
+- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools)
 - 🗲 Quick start:
   - create a room-local agent: `!bai agent create-room-local together-ai my-together-ai-agent`
   - create a global agent: `!bai agent create-global together-ai my-together-ai-agent`

docs/sample-provider-configs/anthropic.yml

@@ -1,8 +1,8 @@
 base_url: https://api.anthropic.com/v1
 api_key: YOUR_API_KEY_HERE
 text_generation:
-  model_id: claude-3-5-sonnet-20240620
-  prompt: You are a brief, but helpful bot.
+  model_id: claude-3-7-sonnet-20250219
+  prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
   temperature: 1.0
   max_response_tokens: 8192
   max_context_tokens: 204800

docs/sample-provider-configs/groq.yml

@@ -2,7 +2,7 @@ base_url: https://api.groq.com/openai/v1
 api_key: YOUR_API_KEY_HERE
 text_generation:
   model_id: llama3-70b-8192
-  prompt: You are a brief, but helpful bot.
+  prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
   temperature: 1.0
   max_response_tokens: 4096
   max_context_tokens: 131072

docs/sample-provider-configs/localai.yml

@@ -2,7 +2,7 @@ base_url: http://my-localai-self-hosted-service:8080/v1
 api_key: YOUR_API_KEY_HERE
 text_generation:
   model_id: gpt-4
-  prompt: You are a brief, but helpful bot.
+  prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
   temperature: 1.0
   max_response_tokens: 4096
   max_context_tokens: 128000

docs/sample-provider-configs/mistral.yml

@@ -2,7 +2,7 @@ base_url: https://api.mistral.ai/v1
 api_key: YOUR_API_KEY_HERE
 text_generation:
   model_id: mistral-large-latest
-  prompt: You are a brief, but helpful bot.
+  prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
   temperature: 1.0
   max_response_tokens: 4096
   max_context_tokens: 128000

docs/sample-provider-configs/ollama.yml

@@ -2,7 +2,7 @@ base_url: http://my-ollama-self-hosted-service:11434/v1
 api_key: YOUR_API_KEY_HERE
 text_generation:
   model_id: gemma2:2b
-  prompt: You are a brief, but helpful bot.
+  prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
   temperature: 1.0
   max_response_tokens: 4096
   max_context_tokens: 128000

docs/sample-provider-configs/openai-compatible.yml

@@ -2,7 +2,7 @@ base_url: ''
 api_key: YOUR_API_KEY_HERE
 text_generation:
   model_id: some-model
-  prompt: You are a brief, but helpful bot.
+  prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
   temperature: 1.0
   max_response_tokens: 4096
   max_context_tokens: 128000

docs/sample-provider-configs/openai.yml

@@ -1,11 +1,18 @@
 base_url: https://api.openai.com/v1
 api_key: YOUR_API_KEY_HERE
 text_generation:
-  model_id: gpt-4o-2024-08-06
-  prompt: You are a brief, but helpful bot.
+  model_id: gpt-5.2
+  prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
   temperature: 1.0
-  max_response_tokens: 16384
-  max_context_tokens: 128000
+  # Reasoning models need to use `max_completion_tokens` instead of `max_response_tokens`.
+  # If you're dealing with a non-reasoning model, specify `max_response_tokens` and unset `max_completion_tokens`.
+  max_response_tokens: null
+  max_completion_tokens: 128000
+  max_context_tokens: 400000
+  # Built-in tools
+  tools:
+    web_search: false
+    code_interpreter: false
 speech_to_text:
   model_id: whisper-1
 text_to_speech:
@@ -14,7 +21,7 @@ text_to_speech:
   speed: 1.0
   response_format: opus
 image_generation:
-  model_id: dall-e-3
-  style: vivid
-  size: 1024x1024
-  quality: standard
+  model_id: gpt-image-1.5
+  style: null
+  size: null
+  quality: null

docs/sample-provider-configs/openrouter.yml

@@ -2,7 +2,7 @@ base_url: https://openrouter.ai/api/v1
 api_key: YOUR_API_KEY_HERE
 text_generation:
   model_id: mattshumer/reflection-70b:free
-  prompt: You are a brief, but helpful bot.
+  prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
   temperature: 1.0
   max_response_tokens: 2048
   max_context_tokens: 8192

docs/sample-provider-configs/together-ai.yml

@@ -2,7 +2,7 @@ base_url: https://api.together.xyz/v1
 api_key: YOUR_API_KEY_HERE
 text_generation:
   model_id: meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo
-  prompt: You are a brief, but helpful bot.
+  prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
   temperature: 1.0
   max_response_tokens: 2048
   max_context_tokens: 8192

docs/usage.md

@@ -11,10 +11,13 @@ This is related to the [💬 Text Generation](./features.md#-text-generation) fe
 
 If there's a text-generation handler agent configured, the bot **may** respond to messages sent in the room.
 
-🖼️ See screenshots of:
+Some models also support vision, so you may be able to mix text and images in the same conversation.
 
-- the [default Text Generation flow](./screenshots/text-generation.webp) for 1:1 rooms
-- the [Text Generation flow in multi-user rooms](./screenshots/text-generation-prefix-requirement.webp) (where the [🗟 Prefix Requirement](./configuration/text-generation.md#-prefix-requirement-type) setting is auto-configured to "required")
+See screenshots of:
+
+- 🖼️ [the default Text Generation flow](./screenshots/text-generation.webp) in 1:1 rooms
+- 🖼️ [the Text Generation flow in multi-user rooms](./screenshots/text-generation-prefix-requirement.webp) (where the [🗟 Prefix Requirement](./configuration/text-generation.md#-prefix-requirement-type) setting is auto-configured to "required")
+- the [on-demand involvement](./features.md#on-demand-involvement) feature
 
 Whether the bot responds depends on:
 
@@ -24,9 +27,9 @@ Whether the bot responds depends on:
 
 - (🎨 agent capabilities) whether the configured `text-generation` (or `catch-all`) handler agent actually supports text-generation. The provider may lack support for this feature or it may be disabled in the [🤖 agents](./agents.md) configuration
 
-- (the [🗟 Prefix Requirement](./configuration/text-generation.md#-prefix-requirement-type) setting) whether a prefix (e.g. `!bai`) is required in front of messages sent to the room. For multi-user rooms, this setting defaults to "required"
+- (the [🗟 Prefix Requirement](./configuration/text-generation.md#-prefix-requirement-type) setting) whether a prefix (e.g. `!bai`) or user mention (e.g. `@baibot`) is required for messages sent to the room. For multi-user rooms, this setting defaults to "required". See [🌟 Features / 💬 Text Generation / On-demand involvement](./features.md#on-demand-involvement) for details.
 
-Room messages start a threaded conversation where you can continue back-and-forth communication with the bot.
+Room messages start a threaded conversation where you can continue back-and-forth communication with the bot. Using [on-demand involvement](./features.md#on-demand-involvement), you can can also mention the bot to provoke it to get involved in any conversation thread or reply chain.
 
 Unless you've enabled the [♻️ Context Management](./features.md#️-context-management) feature, all messages will be sent to the agent's API each time. If the context management feature is enabled, older messages may be dropped.
 
@@ -63,34 +66,48 @@ The speech-to-text feature triggers automatically by default, but can be adjuste
 If all your messages are in the same language, you can improve accuracy & latency by configuring the language (see [🦻 Speech-to-Text / 🔤 Language](./configuration/speech-to-text.md#-language)).
 
 
-### 🖌️ Image Generation
-
-This is related to the [🖌️ Image Generation](./features.md#️-image-generation) feature.
+### Image Generation
 
 This feature is not configurable at the moment. The configuration (size, quality, style) specified at the [🤖 agent](./agents.md) level will be used.
 
+Capabilities depend on the [☁️ provider](./providers.md) and model used.
 
-#### Generating images
 
-Simply send a command like `!bai image A beautiful sunset over the ocean` and the bot will start a threaded conversation and post an image based on your prompt.
+#### 🖌️ Creating images
 
-See a [🖼️ Screenshot of the Image Generation feature](./screenshots/image-generation.webp).
+Simply send a command like `!bai image create A beautiful sunset over the ocean` and the bot will start a threaded conversation and post an image based on your prompt.
 
-You can then, respond in the same message thread with:
+See a [🖼️ Screenshot of the Image Creation feature](./screenshots/image-creation.webp).
+
+You can then respond in the same message thread with:
 
 - more messages, to add more criteria to your prompt.
 - a message saying `again`, to generate one more image with the current prompt.
 
 
-#### Generating stickers
+#### 🎨 Editing images
 
-A variation of [generating images](#generating-images) is to generate "sticker images".
+Simply send a command like `!bai image edit Turn the following image into an anime-style drawing` and the bot will start a threaded conversation asking for more details.
 
-See a [🖼️ Screenshot of the Sticker Generation feature](./screenshots/sticker-generation.webp).
+See a [🖼️ Screenshot of the Image Editing feature (manipulating a single image)](./screenshots/image-editing-single-image.webp) and a [🖼️ Screenshot of the Image Editing feature (manipulating multiple images)](./screenshots/image-editing-multiple-images.webp).
 
-To generate a sticker, send a command like `!bai sticker A huge ramen bowl with lots of chashu and a mountain of beansprouts on top`.
+You can then respond in the same message thread with:
 
-The difference from [generating images](#generating-images) is that the bot will:
+- more messages, to add more criteria to your prompt.
+- one or more images, to provide the images that the bot will operate on.
+- a message saying `go`, to start the image generation process.
+- a message saying `again`, to prompt the bot to generate one more image edit with the current prompt.
+
+
+#### 🫵 Creating stickers
+
+A variation of [creating images](#creating-images) is creating "sticker images".
+
+See a [🖼️ Screenshot of the Sticker Creation feature](./screenshots/sticker-generation.webp).
+
+To create a sticker, send a command like `!bai sticker A huge ramen bowl with lots of chashu and a mountain of beansprouts on top`.
+
+The difference from [creating images](#creating-images) is that the bot will:
 
 - generate a smaller-resolution image (currently hardcoded to `256x256`) - smaller/quicker, but still good enough for a sticker
 - potentially switch to a different (cheaper or otherwise more suitable) model, if available

etc/app/config.yml.dist

@@ -1,16 +1,31 @@
 homeserver:
   # The canonical homeserver domain name
-  server_name: synapse.127.0.0.1.nip.io
-  url: http://synapse.127.0.0.1.nip.io:42020
+  server_name: __HOMESERVER_SERVER_NAME__
+  url: __HOMESERVER_URL__
 
 user:
   mxid_localpart: baibot
+
+  # Authentication: set EITHER password OR access_token + device_id.
+  #
+  # Password-based login (traditional homeservers):
   password: baibot
 
+  # Access token login (for MAS/OIDC-enabled homeservers):
+  # Generate a token via: mas-cli manage issue-compatibility-token <username> [device_id]
+  # access_token: null
+  # device_id: null
+
   # The name the bot uses as a display name and when it refers to itself.
   # Leave empty to use the default (baibot).
   name: baibot
 
+  # An optional path to an image file to be used as a custom avatar image.
+  # - null or empty string: use the default avatar
+  # - "keep": don't touch the avatar, keep whatever is already set
+  # - any other value: path to a custom avatar image file
+  avatar: null
+
   encryption:
     # An optional passphrase to use for backing up and recovering the bot's encryption keys.
     # You can use any string here.
@@ -32,10 +47,14 @@ user:
 # Command prefix. Leave empty to use the default (!bai).
 command_prefix: "!bai"
 
+room:
+  # Whether the bot should send an introduction message after joining a room.
+  post_join_self_introduction_enabled: true
+
 access:
   # Space-separated list of MXID patterns which specify who is an admin.
   admin_patterns:
-    - "@admin:synapse.127.0.0.1.nip.io"
+    - "@admin:__HOMESERVER_SERVER_NAME__"
 
 persistence:
   # This is unset here, because we expect the configuration to come from an environment variable (BAIBOT_PERSISTENCE_DATA_DIR_PATH).
@@ -72,11 +91,18 @@ agents:
     #     base_url: https://api.openai.com/v1
     #     api_key: ""
     #     text_generation:
-    #       model_id: gpt-4o-2024-08-06
-    #       prompt: You are a brief, but helpful bot.
+    #       model_id: gpt-5.2
+    #       prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
     #       temperature: 1.0
-    #       max_response_tokens: 16384
-    #       max_context_tokens: 128000
+    #       # Reasoning models need to use `max_completion_tokens` instead of `max_response_tokens`.
+    #       # If you're dealing with a non-reasoning model, specify `max_response_tokens` and unset `max_completion_tokens`.
+    #       max_response_tokens: null
+    #       max_completion_tokens: 128000
+    #       max_context_tokens: 400000
+    #       # Built-in tools
+    #       tools:
+    #         web_search: false
+    #         code_interpreter: false
     #     speech_to_text:
     #       model_id: whisper-1
     #     text_to_speech:
@@ -85,10 +111,10 @@ agents:
     #       speed: 1.0
     #       response_format: opus
     #     image_generation:
-    #       model_id: dall-e-3
-    #       style: vivid
-    #       size: 1024x1024
-    #       quality: standard
+    #       model_id: gpt-image-1.5
+    #       style: null
+    #       size: null
+    #       quality: null
     #
     # - id: localai
     #   provider: localai
@@ -97,7 +123,7 @@ agents:
     #     api_key: null
     #     text_generation:
     #       model_id: gpt-4
-    #       prompt: You are a brief, but helpful bot.
+    #       prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
     #       temperature: 1.0
     #       max_response_tokens: 16384
     #       max_context_tokens: 128000
@@ -122,7 +148,7 @@ agents:
     #     api_key: null
     #     text_generation:
     #       model_id: "gemma2:2b"
-    #       prompt: "You are an assistant based on the gemma2:2b model. Be brief in your responses."
+    #       prompt: "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
     #       temperature: 1.0
     #       max_response_tokens: 4096
     #       max_context_tokens: 128000
@@ -140,7 +166,7 @@ initial_global_config:
   # Space-separated list of MXID patterns which specify who can use the bot.
   # By default, we let anyone on the homeserver use the bot.
   user_patterns:
-    - "@*:synapse.127.0.0.1.nip.io"
+    - "@*:__HOMESERVER_SERVER_NAME__"
 
 # Controls logging.
 #

etc/services/continuwuity/compose.yml

@@ -0,0 +1,23 @@
+services:
+  continuwuity:
+    image: forgejo.ellis.link/continuwuation/continuwuity:v0.5.6
+    user: "${UID}:${GID}"
+    restart: unless-stopped
+    cap_drop:
+      - ALL
+    read_only: true
+    environment:
+      CONDUWUIT_CONFIG: /etc/continuwuity/continuwuity.toml
+      CONDUWUIT_DATABASE_PATH: /var/lib/continuwuity
+    ports:
+      - "${SERVICE_CONTINUWUITY_BIND_PORT_CLIENT_API}:6167"
+    volumes:
+      - ../../etc/services/continuwuity/config:/etc/continuwuity:ro
+      - ./continuwuity/data:/var/lib/continuwuity
+    tmpfs:
+      - /tmp:rw,noexec,nosuid,size=500m
+
+networks:
+  default:
+    name: ${NETWORK_NAME}
+    external: true

etc/services/continuwuity/config/continuwuity.toml

@@ -0,0 +1,19 @@
+[global]
+server_name = "continuwuity.127.0.0.1.nip.io"
+
+address = "0.0.0.0"
+port = 6167
+
+database_path = "/var/lib/continuwuity"
+
+allow_registration = true
+yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = true
+
+new_user_displayname_suffix = ""
+
+max_request_size = 20_000_000
+
+allow_federation = false
+trusted_servers = ["matrix.org"]
+
+log = "info,state_res=warn,rocket=off,_=off,sled=off"

etc/services/continuwuity/register-user.sh

@@ -0,0 +1,48 @@
+#!/bin/sh
+set -eu
+
+if [ $# -ne 3 ]; then
+	echo "Usage: $0 <env-file> <username> <password>"
+	exit 1
+fi
+
+ENV_FILE="$1"
+USERNAME="$2"
+PASSWORD="$3"
+
+SERVER="http://$(grep '^SERVICE_CONTINUWUITY_BIND_PORT_CLIENT_API=' "${ENV_FILE}" | cut -d= -f2)"
+REGISTER_URL="${SERVER}/_matrix/client/v3/register"
+
+echo "Registering user '${USERNAME}' on ${SERVER}..."
+
+SESSION_RESPONSE=$(curl -s -X POST "${REGISTER_URL}" \
+	-H 'Content-Type: application/json' \
+	-d "{\"username\": \"${USERNAME}\", \"password\": \"${PASSWORD}\"}")
+
+SESSION_ID=$(echo "${SESSION_RESPONSE}" | grep -o '"session":"[^"]*"' | head -1 | cut -d'"' -f4)
+if [ -z "${SESSION_ID}" ]; then
+	echo "Error: Could not get session ID. Response: ${SESSION_RESPONSE}"
+	exit 1
+fi
+
+# Determine the required auth flow from the server response.
+# The first user requires m.login.registration_token (bootstrap token from logs).
+# Subsequent users use m.login.dummy (open registration).
+if echo "${SESSION_RESPONSE}" | grep -q 'm.login.registration_token'; then
+	CONTAINER_ID=$(docker ps -q --filter name=baibot-continuwuity-continuwuity)
+	REG_TOKEN=$(docker logs "${CONTAINER_ID}" 2>&1 | sed 's/\x1b\[[0-9;]*m//g' | grep 'using the registration token' | grep -oP 'registration token \K[A-Za-z0-9]+' | head -1)
+	AUTH_BODY="{\"type\": \"m.login.registration_token\", \"token\": \"${REG_TOKEN}\", \"session\": \"${SESSION_ID}\"}"
+else
+	AUTH_BODY="{\"type\": \"m.login.dummy\", \"session\": \"${SESSION_ID}\"}"
+fi
+
+RESULT=$(curl -s -X POST "${REGISTER_URL}" \
+	-H 'Content-Type: application/json' \
+	-d "{\"username\": \"${USERNAME}\", \"password\": \"${PASSWORD}\", \"auth\": ${AUTH_BODY}}")
+
+if echo "${RESULT}" | grep -q '"user_id"'; then
+	echo "Successfully registered user: $(echo "${RESULT}" | grep -o '"user_id":"[^"]*"' | cut -d'"' -f4)"
+else
+	echo "Registration failed. Response: ${RESULT}"
+	exit 1
+fi

etc/services/core/element-web/nginx.conf

@@ -1,60 +0,0 @@
-# This is a custom nginx configuration file that we use in the container (instead of the default one),
-# because it allows us to run nginx with a non-root user.
-#
-# For this to work, the default vhost file (`/etc/nginx/conf.d/default.conf`) also needs to be removed.
-# (mounting `/dev/null` over `/etc/nginx/conf.d/default.conf` works well)
-#
-# The following changes have been done compared to a default nginx configuration file:
-# - default server port is changed (80 -> 8080), so that a non-root user can bind it
-# - various temp paths are changed to `/tmp`, so that a non-root user can write to them
-# - the `user` directive was removed, as we don't want nginx to switch users
-
-worker_processes  1;
-
-error_log /var/log/nginx/error.log warn;
-pid /tmp/nginx.pid;
-
-
-events {
-	worker_connections  1024;
-}
-
-
-http {
-	client_body_temp_path /tmp/client_body_temp;
-	proxy_temp_path /tmp/proxy_temp;
-	fastcgi_temp_path /tmp/fastcgi_temp;
-	uwsgi_temp_path /tmp/uwsgi_temp;
-	scgi_temp_path /tmp/scgi_temp;
-
-	include       /etc/nginx/mime.types;
-	default_type  application/octet-stream;
-
-	log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
-					  '$status $body_bytes_sent "$http_referer" '
-					  '"$http_user_agent" "$http_x_forwarded_for"';
-
-	access_log  /var/log/nginx/access.log  main;
-
-	sendfile        on;
-	#tcp_nopush     on;
-
-	keepalive_timeout  65;
-
-	#gzip  on;
-
-	server {
-		listen       8080;
-		server_name  localhost;
-
-		location / {
-			root   /usr/share/nginx/html;
-			index  index.html index.htm;
-		}
-
-		error_page   500 502 503 504  /50x.html;
-		location = /50x.html {
-			root   /usr/share/nginx/html;
-		}
-	}
-}

etc/services/element-web/compose.yml

@@ -0,0 +1,21 @@
+services:
+  element-web:
+    image: ghcr.io/element-hq/element-web:v1.12.11
+    user: "${UID}:${GID}"
+    restart: unless-stopped
+    environment:
+      ELEMENT_WEB_PORT: 8080
+    ports:
+      - "${SERVICE_ELEMENT_WEB_BIND_PORT_HTTP}:8080"
+    volumes:
+    - ./element-web/config.json:/app/config.json:ro
+    tmpfs:
+      - /var/cache/nginx:rw,mode=777
+      - /var/run:rw,mode=777
+      - /tmp/element-web-config:rw,mode=777
+      - /etc/nginx/conf.d:rw,mode=777
+
+networks:
+  default:
+    name: ${NETWORK_NAME}
+    external: true

etc/services/element-web/config.json.dist

@@ -1,9 +1,9 @@
 {
-    "default_hs_url": "http://synapse.127.0.0.1.nip.io:42020",
+    "default_hs_url": "__HOMESERVER_CLIENT_URL__",
     "default_is_url": "https://vector.im",
     "integrations_ui_url": "https://scalar.vector.im/",
     "integrations_rest_url": "https://scalar.vector.im/api",
-    "bug_report_endpoint_url": "https://riot.im/bugreports/submit",
+    "bug_report_endpoint_url": "https://element.io/bugreports/submit",
     "enableLabs": true,
     "roomDirectory": {
         "servers": [

etc/services/env.dist

@@ -3,6 +3,8 @@ SERVICE_SYNAPSE_BIND_PORT_FEDERATION_API=127.0.0.1:42028
 
 SERVICE_ELEMENT_WEB_BIND_PORT_HTTP=127.0.0.1:42025
 
+SERVICE_CONTINUWUITY_BIND_PORT_CLIENT_API=127.0.0.1:42030
+
 SERVICE_OLLAMA_BIND_PORT_HTTP=127.0.0.1:42026
 
 # See https://localai.io/basics/container/#all-in-one-images for the list of available images

etc/services/ollama/compose.yml

@@ -1,6 +1,6 @@
 services:
   ollama:
-    image: docker.io/ollama/ollama:0.3.9
+    image: docker.io/ollama/ollama:0.17.6
     restart: unless-stopped
     ports:
       - "${SERVICE_OLLAMA_BIND_PORT_HTTP}:11434"

etc/services/synapse/compose.yml

@@ -1,6 +1,6 @@
 services:
   postgres:
-    image: docker.io/postgres:16.3-alpine
+    image: docker.io/postgres:18.3-alpine
     user: ${UID}:${GID}
     restart: unless-stopped
     environment:
@@ -8,12 +8,13 @@ services:
      POSTGRES_PASSWORD: synapse-password
      POSTGRES_DB: homeserver
      POSTGRES_INITDB_ARGS: --lc-collate C --lc-ctype C --encoding UTF8
+     PGDATA: /data
     volumes:
-    - ./postgres:/var/lib/postgresql/data
+    - ./postgres:/data
     - /etc/passwd:/etc/passwd:ro
 
   synapse:
-    image: ghcr.io/element-hq/synapse:v1.114.0
+    image: ghcr.io/element-hq/synapse:v1.148.0
     user: "${UID}:${GID}"
     restart: unless-stopped
     entrypoint: python
@@ -22,19 +23,9 @@ services:
     - "${SERVICE_SYNAPSE_BIND_PORT_CLIENT_API}:8008"
     - "${SERVICE_SYNAPSE_BIND_PORT_FEDERATION_API}:8008"
     volumes:
-    - ../../etc/services/core/synapse/config:/config:ro
+    - ../../etc/services/synapse/config:/config:ro
     - ./synapse/media-store:/media-store
 
-  element-web:
-    image: docker.io/vectorim/element-web:v1.11.77
-    user: "${UID}:${GID}"
-    restart: unless-stopped
-    ports:
-      - "${SERVICE_ELEMENT_WEB_BIND_PORT_HTTP}:8080"
-    volumes:
-    - ../../etc/services/core/element-web/nginx.conf:/etc/nginx/nginx.conf:ro
-    - ../../etc/services/core/element-web/config.json:/app/config.json:ro
-
 networks:
   default:
     name: ${NETWORK_NAME}

etc/services/synapse/config/homeserver.yaml

@@ -579,7 +579,7 @@ rc_login:
 #
 #federation_rr_transactions_per_room_per_second: 50
 
-
+enable_authenticated_media: true
 
 # Directory where uploaded images and attachments are stored.
 #

justfile

@@ -2,10 +2,33 @@ project_name := "baibot"
 container_image_name := "localhost/baibot"
 project_container_network := "baibot"
 
+admin_username := "admin"
+admin_password := "admin"
+bot_username := "baibot"
+bot_password := "baibot"
+
+homeserver := `cat var/homeserver 2>/dev/null || echo continuwuity`
+
+mise_data_dir := env("MISE_DATA_DIR", justfile_directory() / "var/mise")
+mise_trusted_config_paths := justfile_directory() / "mise.toml"
+
 # Show help by default
 default:
 	@just --list --justfile {{ justfile() }}
 
+# Selects which homeserver implementation to use (continuwuity or synapse)
+homeserver-init value:
+	#!/bin/sh
+	mkdir -p {{ justfile_directory() }}/var
+	echo {{ value }} > {{ justfile_directory() }}/var/homeserver
+	echo ""
+	echo "⚠️  If you had already prepared your app configuration (var/app/local/config.yml or var/app/container/config.yml),"
+	echo "    you will need to update it manually or delete it and re-run the prepare step."
+	echo "    You should also delete var/app/local/data and/or var/app/container/data,"
+	echo "    as old application state is not compatible across homeserver implementations."
+	echo ""
+	echo "⚠️  If Element Web was already prepared, delete var/services/element-web/ to regenerate its config."
+
 # Builds and runs a development binary
 run-locally *extra_args: app-local-prepare
 	RUST_BACKTRACE=1 \
@@ -13,7 +36,8 @@ run-locally *extra_args: app-local-prepare
 	BAIBOT_PERSISTENCE_DATA_DIR_PATH={{ justfile_directory() }}/var/app/local/data \
 	cargo run -- {{ extra_args }}
 
-run-in-container *extra_args: app-container-prepare build-container-image
+# Builds and runs the bot in a container
+run-in-container *extra_args: app-container-prepare build-container-image-debug
 	/usr/bin/env docker run \
 	-it \
 	--rm \
@@ -25,12 +49,16 @@ run-in-container *extra_args: app-container-prepare build-container-image
 	--env BAIBOT_PERSISTENCE_DATA_DIR_PATH=/data \
 	--mount type=bind,src={{ justfile_directory() }}/var/app/container/config.yml,dst=/app/config.yml,ro \
 	--mount type=bind,src={{ justfile_directory() }}/var/app/container/data,dst=/data \
-	{{ container_image_name }} {{ extra_args }}
+	{{ container_image_name }}:latest {{ extra_args }}
 
 # Runs tests
 test *extra_args:
 	RUST_BACKTRACE=1 cargo test {{ extra_args }}
 
+# Formats the code
+fmt:
+	RUST_BACKTRACE=1 cargo fmt --all
+
 # Builds a debug binary (target/debug/*)
 build-debug *extra_args:
 	RUST_BACKTRACE=1 cargo build {{ extra_args }}
@@ -38,9 +66,16 @@ build-debug *extra_args:
 # Builds an optimized release binary (target/release/*)
 build-release *extra_args: (build-debug "--release")
 
-# Builds a container image
-build-container-image tag='latest':
+# Builds a container image (debug mode)
+build-container-image-debug tag='latest': (_build-container-image "false" tag)
+
+# Builds a container image (release mode)
+build-container-image-release tag='latest': (_build-container-image "true" tag)
+
+_build-container-image release_build tag:
 	/usr/bin/env docker build \
+	--build-arg RELEASE_BUILD={{ release_build }} \
+	-f {{ justfile_directory() }}/Dockerfile \
 	-t {{ container_image_name }}:{{ tag }} \
 	.
 
@@ -53,9 +88,13 @@ docker-compose services_type *extra_args:
 	-p {{ project_name }}-{{ services_type }} \
 	{{ extra_args }}
 
-# Runs a docker-compose command against the core services
-docker-compose-core *extra_args:
-	just docker-compose core {{ extra_args }}
+# Runs a docker-compose command against the synapse services
+docker-compose-synapse *extra_args:
+	just docker-compose synapse {{ extra_args }}
+
+# Runs a docker-compose command against the element-web services
+docker-compose-element-web *extra_args:
+	just docker-compose element-web {{ extra_args }}
 
 # Runs a docker-compose command against the localai services
 docker-compose-localai *extra_args:
@@ -65,17 +104,52 @@ docker-compose-localai *extra_args:
 docker-compose-ollama *extra_args:
 	just docker-compose ollama {{ extra_args }}
 
-# Runs all core dependency components (in the background)
-services-start: services-prepare (docker-compose-core "up" "-d")
+# Runs a docker-compose command against the continuwuity services
+docker-compose-continuwuity *extra_args:
+	just docker-compose continuwuity {{ extra_args }}
 
-# Stops all core dependency components
-services-stop: (docker-compose-core "down")
+# Runs the homeserver and Element Web (in the background)
+services-start: services-prepare
+	just -f {{ justfile_directory() }}/justfile {{ homeserver }}-start
+	just -f {{ justfile_directory() }}/justfile element-web-start
 
-# Tails the logs for all running core services
-services-tail-logs: (docker-compose-core "logs" "-f")
+# Stops Element Web and the homeserver
+services-stop:
+	just -f {{ justfile_directory() }}/justfile element-web-stop
+	just -f {{ justfile_directory() }}/justfile {{ homeserver }}-stop
 
-# Prepares the core services for running
-services-prepare: _prepare-var-services-env _prepare-var-services-postgres _prepare-var-services-synapse _prepare-container-network
+# Tails the logs for the homeserver and Element Web
+services-tail-logs:
+	just -f {{ justfile_directory() }}/justfile {{ homeserver }}-tail-logs
+
+# Prepares the homeserver and Element Web for running
+services-prepare:
+	just -f {{ justfile_directory() }}/justfile {{ homeserver }}-prepare
+	just -f {{ justfile_directory() }}/justfile element-web-prepare
+
+# Runs Synapse (in the background)
+synapse-start: synapse-prepare (docker-compose-synapse "up" "-d")
+
+# Stops Synapse
+synapse-stop: (docker-compose-synapse "down")
+
+# Tails the logs for Synapse
+synapse-tail-logs: (docker-compose-synapse "logs" "-f")
+
+# Prepares Synapse for running
+synapse-prepare: _prepare-var-services-env _prepare-var-services-postgres _prepare-var-services-synapse _prepare-container-network
+
+# Runs Element Web (in the background)
+element-web-start: element-web-prepare (docker-compose-element-web "up" "-d")
+
+# Stops Element Web
+element-web-stop: (docker-compose-element-web "down")
+
+# Tails the logs for Element Web
+element-web-tail-logs: (docker-compose-element-web "logs" "-f")
+
+# Prepares Element Web for running
+element-web-prepare: _prepare-var-services-env _prepare-var-services-element-web _prepare-container-network
 
 # Runs LocalAI (in the background)
 localai-start: localai-prepare (docker-compose-localai "up" "-d")
@@ -101,6 +175,27 @@ ollama-tail-logs: (docker-compose-ollama "logs" "-f")
 # Prepares Ollama for running
 ollama-prepare: _prepare-var-services-env _prepare-var-services-ollama _prepare-container-network
 
+# Runs Continuwuity (in the background)
+continuwuity-start: continuwuity-prepare (docker-compose-continuwuity "up" "-d")
+
+# Stops Continuwuity
+continuwuity-stop: (docker-compose-continuwuity "down")
+
+# Tails the logs for Continuwuity
+continuwuity-tail-logs: (docker-compose-continuwuity "logs" "-f")
+
+# Prepares Continuwuity for running
+continuwuity-prepare: _prepare-var-services-env _prepare-var-services-continuwuity _prepare-container-network
+
+# Registers a user on Continuwuity via the Matrix Client-Server API
+continuwuity-register-user username password:
+	{{ justfile_directory() }}/etc/services/continuwuity/register-user.sh {{ justfile_directory() }}/var/services/env {{ username }} {{ password }}
+
+# Prepares the Continuwuity user accounts
+continuwuity-users-prepare: continuwuity-prepare
+	just -f {{ justfile_directory() }}/justfile continuwuity-register-user "{{ admin_username }}" "{{ admin_password }}"
+	just -f {{ justfile_directory() }}/justfile continuwuity-register-user "{{ bot_username }}" "{{ bot_password }}"
+
 # Pulls an Ollama model
 ollama-pull-model model_id:
 	just -f {{ justfile_directory() }}/justfile docker-compose-ollama \
@@ -114,16 +209,20 @@ app-local-prepare: _prepare-var-app-local-config_yml _prepare-var-app-local-data
 app-container-prepare: _prepare-var-app-container-config_yml _prepare-var-app-container-data
 
 # Prepares the user accounts
-users-prepare: services-prepare
-	just -f {{ justfile_directory() }}/justfile synapse-register-admin-user "admin" "admin"
-	just -f {{ justfile_directory() }}/justfile synapse-register-regular-user "baibot" "baibot"
+users-prepare:
+	just -f {{ justfile_directory() }}/justfile {{ homeserver }}-users-prepare
+
+# Prepares the Synapse user accounts
+synapse-users-prepare: synapse-prepare
+	just -f {{ justfile_directory() }}/justfile synapse-register-admin-user "{{ admin_username }}" "{{ admin_password }}"
+	just -f {{ justfile_directory() }}/justfile synapse-register-regular-user "{{ bot_username }}" "{{ bot_password }}"
 
 # Starts a Postgres CLI (psql)
-postgres-cli: services-prepare (docker-compose-core "exec" "postgres" "/bin/sh" "-c" "'PGUSER=synapse PGPASSWORD=synapse-password PGDATABASE=homeserver psql -h postgres'")
+postgres-cli: synapse-prepare (docker-compose-synapse "exec" "postgres" "/bin/sh" "-c" "'PGUSER=synapse PGPASSWORD=synapse-password PGDATABASE=homeserver psql -h postgres'")
 
-# Creates an administrator user
-synapse-register-admin-user username password: services-prepare
-	just -f {{ justfile_directory() }}/justfile docker-compose-core \
+# Creates an administrator user on Synapse
+synapse-register-admin-user username password: synapse-prepare
+	just -f {{ justfile_directory() }}/justfile docker-compose-synapse \
 		exec synapse \
 		register_new_matrix_user \
 		--admin \
@@ -132,9 +231,9 @@ synapse-register-admin-user username password: services-prepare
 		-c /config/homeserver.yaml \
 		http://localhost:8008
 
-# Create a regular user
-synapse-register-regular-user username password: services-prepare
-	just -f {{ justfile_directory() }}/justfile docker-compose-core \
+# Creates a regular user on Synapse
+synapse-register-regular-user username password: synapse-prepare
+	just -f {{ justfile_directory() }}/justfile docker-compose-synapse \
 		exec synapse \
 		register_new_matrix_user \
 		--no-admin \
@@ -147,6 +246,44 @@ synapse-register-regular-user username password: services-prepare
 clippy *extra_args:
 	cargo clippy {{ extra_args }}
 
+# Checks that the code compiles without building
+check:
+	cargo check
+
+# Invokes mise with the project-local data directory
+mise *args: _ensure_mise_data_directory
+	#!/bin/sh
+	export MISE_DATA_DIR="{{ mise_data_dir }}"
+	export MISE_TRUSTED_CONFIG_PATHS="{{ mise_trusted_config_paths }}"
+	mise {{ args }}
+
+# Runs prek (pre-commit hooks manager) with the given arguments
+prek *args: _ensure_mise_tools_installed
+	@just --justfile {{ justfile() }} mise exec -- prek {{ args }}
+
+# Runs pre-commit hooks on staged files
+prek-run-on-staged *args: _ensure_mise_tools_installed
+	@just --justfile {{ justfile() }} mise exec -- prek run {{ args }}
+
+# Runs pre-commit hooks on all files
+prek-run-on-all *args: _ensure_mise_tools_installed
+	@just --justfile {{ justfile() }} mise exec -- prek run --all-files {{ args }}
+
+# Installs the git pre-commit hook (runs prek automatically before each commit)
+prek-install-git-pre-commit-hook: _ensure_mise_tools_installed
+	@just --justfile {{ justfile() }} mise exec -- prek install
+
+# Internal - ensures var/mise directory exists
+_ensure_mise_data_directory:
+	#!/bin/sh
+	if [ ! -d "{{ mise_data_dir }}" ]; then
+		mkdir -p "{{ mise_data_dir }}"
+	fi
+
+# Internal - ensures mise tools are installed
+_ensure_mise_tools_installed: _ensure_mise_data_directory
+	@just --justfile {{ justfile() }} mise install --quiet
+
 _prepare-var-services-env:
 	#!/bin/sh
 	cd {{ justfile_directory() }};
@@ -176,6 +313,22 @@ _prepare-var-services-synapse:
 		mkdir -p var/services/synapse/media-store
 	fi
 
+_prepare-var-services-element-web:
+	#!/bin/sh
+	cd {{ justfile_directory() }};
+
+	if [ ! -f var/services/element-web/config.json ]; then
+		mkdir -p var/services/element-web
+		cp {{ justfile_directory() }}/etc/services/element-web/config.json.dist var/services/element-web/config.json
+
+		homeserver="{{ homeserver }}"
+		if [ "$homeserver" = "continuwuity" ]; then
+			sed --in-place 's|__HOMESERVER_CLIENT_URL__|http://continuwuity.127.0.0.1.nip.io:42030|g' var/services/element-web/config.json
+		elif [ "$homeserver" = "synapse" ]; then
+			sed --in-place 's|__HOMESERVER_CLIENT_URL__|http://synapse.127.0.0.1.nip.io:42020|g' var/services/element-web/config.json
+		fi
+	fi
+
 _prepare-var-services-ollama:
 	#!/bin/sh
 	cd {{ justfile_directory() }};
@@ -184,6 +337,14 @@ _prepare-var-services-ollama:
 		mkdir -p var/services/ollama
 	fi
 
+_prepare-var-services-continuwuity:
+	#!/bin/sh
+	cd {{ justfile_directory() }};
+
+	if [ ! -f var/services/continuwuity ]; then
+		mkdir -p var/services/continuwuity/data
+	fi
+
 _prepare-var-services-localai:
 	#!/bin/sh
 	cd {{ justfile_directory() }};
@@ -207,6 +368,15 @@ _prepare-var-app-local-config_yml:
 	if [ ! -f var/app/local/config.yml ]; then
 		mkdir -p var/app/local
 		cp {{ justfile_directory() }}/etc/app/config.yml.dist var/app/local/config.yml
+
+		homeserver="{{ homeserver }}"
+		if [ "$homeserver" = "continuwuity" ]; then
+			sed --in-place 's/__HOMESERVER_SERVER_NAME__/continuwuity.127.0.0.1.nip.io/g' var/app/local/config.yml
+			sed --in-place 's|__HOMESERVER_URL__|http://continuwuity.127.0.0.1.nip.io:42030|g' var/app/local/config.yml
+		elif [ "$homeserver" = "synapse" ]; then
+			sed --in-place 's/__HOMESERVER_SERVER_NAME__/synapse.127.0.0.1.nip.io/g' var/app/local/config.yml
+			sed --in-place 's|__HOMESERVER_URL__|http://synapse.127.0.0.1.nip.io:42020|g' var/app/local/config.yml
+		fi
 	fi
 
 _prepare-var-app-local-data:
@@ -224,7 +394,18 @@ _prepare-var-app-container-config_yml:
 	if [ ! -f var/app/container/config.yml ]; then
 		mkdir -p var/app/container
 		cp {{ justfile_directory() }}/etc/app/config.yml.dist var/app/container/config.yml
-		sed --in-place 's/synapse.127.0.0.1.nip.io:42020/synapse:8008/g' var/app/container/config.yml
+
+		homeserver="{{ homeserver }}"
+		if [ "$homeserver" = "continuwuity" ]; then
+			sed --in-place 's/__HOMESERVER_SERVER_NAME__/continuwuity.127.0.0.1.nip.io/g' var/app/container/config.yml
+			sed --in-place 's|__HOMESERVER_URL__|http://continuwuity.127.0.0.1.nip.io:42030|g' var/app/container/config.yml
+			sed --in-place 's/continuwuity.127.0.0.1.nip.io:42030/continuwuity:6167/g' var/app/container/config.yml
+		elif [ "$homeserver" = "synapse" ]; then
+			sed --in-place 's/__HOMESERVER_SERVER_NAME__/synapse.127.0.0.1.nip.io/g' var/app/container/config.yml
+			sed --in-place 's|__HOMESERVER_URL__|http://synapse.127.0.0.1.nip.io:42020|g' var/app/container/config.yml
+			sed --in-place 's/synapse.127.0.0.1.nip.io:42020/synapse:8008/g' var/app/container/config.yml
+		fi
+
 		sed --in-place 's/127.0.0.1:42026/ollama:11434/g' var/app/container/config.yml
 		sed --in-place 's/127.0.0.1:42027/localai:8080/g' var/app/container/config.yml
 	fi
@@ -236,4 +417,3 @@ _prepare-var-app-container-data:
 	if [ ! -f var/app/container/data ]; then
 		mkdir -p var/app/container/data
 	fi
-

mise.toml

@@ -0,0 +1,6 @@
+[tools]
+prek = "0.3.2"
+
+[settings]
+# Disable automatic trust prompts - we trust this config
+yes = true

renovate.json

@@ -0,0 +1,9 @@
+{
+	"$schema": "https://docs.renovatebot.com/renovate-schema.json",
+	"extends": [
+		"config:recommended"
+	],
+	"labels": [
+		"dependencies"
+	]
+}

src/agent/definition.rs

@@ -33,11 +33,11 @@ pub struct AgentDefinition {
     )]
     pub provider: AgentProvider,
 
-    pub config: serde_yaml::Value,
+    pub config: serde_yaml_ng::Value,
 }
 
 impl AgentDefinition {
-    pub fn new(id: String, provider: AgentProvider, config: serde_yaml::Value) -> Self {
+    pub fn new(id: String, provider: AgentProvider, config: serde_yaml_ng::Value) -> Self {
         Self {
             id,
             provider,

src/agent/instantiation.rs

@@ -1,6 +1,6 @@
 use super::{
-    provider::{self, ControllerType},
     AgentDefinition, AgentProvider, PublicIdentifier,
+    provider::{self, ControllerType},
 };
 
 // Dead-code is allowed. We do not use these enum struct payloads directly,
@@ -15,7 +15,7 @@ pub enum Error {
     // Contains the error from the constructor function
     ConstructionFailed(anyhow::Error),
     // Contains the error from the YAML deserialization function
-    Yaml(serde_yaml::Error),
+    Yaml(serde_yaml_ng::Error),
 }
 
 pub type Result<T> = std::result::Result<T, Error>;
@@ -69,7 +69,7 @@ pub(super) fn create(
 pub fn create_from_provider_and_yaml_value_config(
     provider: &AgentProvider,
     identifier: &PublicIdentifier,
-    config: serde_yaml::Value,
+    config: serde_yaml_ng::Value,
 ) -> Result<AgentInstance> {
     let definition = AgentDefinition::new(identifier.prefixless(), provider.to_owned(), config);
 
@@ -79,7 +79,7 @@ pub fn create_from_provider_and_yaml_value_config(
 fn create_controller_from_provider_and_json_value_config(
     agent_id: &str,
     provider: &AgentProvider,
-    config: serde_yaml::Value,
+    config: serde_yaml_ng::Value,
 ) -> Result<ControllerType> {
     match provider {
         AgentProvider::Anthropic => {
@@ -112,43 +112,43 @@ fn create_controller_from_provider_and_json_value_config(
     }
 }
 
-pub fn default_config_for_provider(provider: &AgentProvider) -> serde_yaml::Value {
+pub fn default_config_for_provider(provider: &AgentProvider) -> serde_yaml_ng::Value {
     match provider {
         AgentProvider::Anthropic => {
             let config = super::provider::anthropic::default_config();
-            serde_yaml::to_value(config).expect("Failed to serialize config")
+            serde_yaml_ng::to_value(config).expect("Failed to serialize config")
         }
         AgentProvider::Groq => {
             let config = super::provider::groq::default_config();
-            serde_yaml::to_value(config).expect("Failed to serialize config")
+            serde_yaml_ng::to_value(config).expect("Failed to serialize config")
         }
         AgentProvider::LocalAI => {
             let config = super::provider::localai::default_config();
-            serde_yaml::to_value(config).expect("Failed to serialize config")
+            serde_yaml_ng::to_value(config).expect("Failed to serialize config")
         }
         AgentProvider::Mistral => {
             let config = super::provider::mistral::default_config();
-            serde_yaml::to_value(config).expect("Failed to serialize config")
+            serde_yaml_ng::to_value(config).expect("Failed to serialize config")
         }
         AgentProvider::Ollama => {
             let config = super::provider::ollama::default_config();
-            serde_yaml::to_value(config).expect("Failed to serialize config")
+            serde_yaml_ng::to_value(config).expect("Failed to serialize config")
         }
         AgentProvider::OpenAI => {
             let config = super::provider::openai::default_config();
-            serde_yaml::to_value(config).expect("Failed to serialize config")
+            serde_yaml_ng::to_value(config).expect("Failed to serialize config")
         }
         AgentProvider::OpenAICompat => {
             let config = super::provider::openai_compat::default_config();
-            serde_yaml::to_value(config).expect("Failed to serialize config")
+            serde_yaml_ng::to_value(config).expect("Failed to serialize config")
         }
         AgentProvider::OpenRouter => {
             let config = super::provider::openrouter::default_config();
-            serde_yaml::to_value(config).expect("Failed to serialize config")
+            serde_yaml_ng::to_value(config).expect("Failed to serialize config")
         }
         AgentProvider::TogetherAI => {
             let config = super::provider::togetherai::default_config();
-            serde_yaml::to_value(config).expect("Failed to serialize config")
+            serde_yaml_ng::to_value(config).expect("Failed to serialize config")
         }
     }
 }

src/agent/manager.rs

@@ -1,7 +1,7 @@
-use super::instantiation;
-use super::instantiation::AgentInstance;
 use super::AgentDefinition;
 use super::PublicIdentifier;
+use super::instantiation;
+use super::instantiation::AgentInstance;
 use crate::entity::RoomConfigContext;
 
 #[derive(Debug)]

src/agent/mod.rs

@@ -11,11 +11,15 @@ pub use manager::Manager;
 
 pub use definition::AgentDefinition;
 
-pub use instantiation::create_from_provider_and_yaml_value_config;
-pub use instantiation::default_config_for_provider;
 pub use instantiation::AgentInstance;
 pub use instantiation::Error as AgentInstantiationError;
 pub use instantiation::Result as AgentInstantiationResult;
+pub use instantiation::create_from_provider_and_yaml_value_config;
+pub use instantiation::default_config_for_provider;
 
 pub use provider::{AgentProvider, AgentProviderInfo, ControllerTrait};
 pub use purpose::AgentPurpose;
+
+pub(super) fn default_prompt() -> &'static str {
+    "You are a brief, but helpful bot called {{ baibot_name }} powered by the {{ baibot_model_id }} model. The date/time of this conversation's start is: {{ baibot_conversation_start_time_utc }}."
+}

src/agent/provider/anthropic/config.rs

@@ -1,8 +1,6 @@
 use serde::{Deserialize, Serialize};
 
-use anthropic_rs::models::claude::ClaudeModel;
-
-use crate::agent::provider::ConfigTrait;
+use crate::agent::{default_prompt, provider::ConfigTrait};
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Config {
@@ -28,6 +26,9 @@ impl ConfigTrait for Config {
         if self.base_url.is_empty() {
             return Err("The base URL must not be empty.".to_owned());
         }
+        if !self.base_url.ends_with("/v1") {
+            return Err("The base URL must end with '/v1'.".to_owned());
+        }
         if self.api_key.is_empty() {
             return Err("The API key must not be empty.".to_owned());
         }
@@ -58,7 +59,7 @@ impl Default for TextGenerationConfig {
     fn default() -> Self {
         Self {
             model_id: default_text_model_id(),
-            prompt: Some("You are a brief, but helpful bot.".to_owned()),
+            prompt: Some(default_prompt().to_owned()),
             temperature: super::super::default_temperature(),
             max_response_tokens: 8192,
             max_context_tokens: 204_800,
@@ -67,5 +68,5 @@ impl Default for TextGenerationConfig {
 }
 
 fn default_text_model_id() -> String {
-    ClaudeModel::Claude35Sonnet.as_str().to_owned()
+    "claude-3-7-sonnet-20250219".to_owned()
 }

src/agent/provider/anthropic/controller.rs

@@ -1,30 +1,28 @@
 use std::fmt::Debug;
-use std::str::FromStr;
 use std::sync::Arc;
 
-use anthropic_rs::completion::message::ContentType;
-use anthropic_rs::{
-    client::Client as AnthropicClient, config::Config as AnthropicConfig,
-    models::claude::ClaudeModel,
-};
+use anthropic::client::{Client, ClientBuilder};
+use anthropic::types::ContentBlock;
 
 use super::super::ControllerTrait;
-use crate::agent::provider::entity::{
-    ImageGenerationResult, PingResult, TextGenerationParams, TextGenerationResult,
-    TextToSpeechParams, TextToSpeechResult,
-};
-use crate::agent::provider::{ImageGenerationParams, SpeechToTextParams, SpeechToTextResult};
 use crate::agent::AgentPurpose;
+use crate::agent::provider::entity::{
+    ImageEditResult, ImageGenerationResult, ImageSource, PingResult, TextGenerationParams,
+    TextGenerationResult, TextToSpeechParams, TextToSpeechResult,
+};
+use crate::agent::provider::{
+    ImageEditParams, ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
+};
 use crate::conversation::llm::{
-    shorten_messages_list_to_context_size, Author as LLMAuthor, Conversation as LLMConversation,
-    Message as LLMMessage,
+    Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
+    MessageContent as LLMMessageContent, shorten_messages_list_to_context_size,
 };
 use crate::strings;
 
 use super::config::Config;
 
 struct ControllerInner {
-    client: AnthropicClient,
+    client: Client,
 }
 
 #[derive(Clone)]
@@ -43,18 +41,20 @@ impl Debug for Controller {
 
 impl Controller {
     pub fn new(config: Config) -> anyhow::Result<Self> {
-        let anthropic_config =
-            AnthropicConfig::new(config.api_key.clone()).with_base_url(config.base_url.clone());
+        // The previous library that we used expected a base URL that ends with "/v1"
+        // (e.g. "https://api.anthropic.com/v1"), while the new one doesn't.
+        //
+        // To keep backward compatibility, we don't ask people to change their configuration
+        // and rather adapt by removing the "/v1" from the base URL.
+        if !config.base_url.ends_with("/v1") {
+            return Err(anyhow::anyhow!("base_url must end with '/v1'"));
+        }
 
-        let client = match AnthropicClient::new(anthropic_config) {
-            Ok(client) => client,
-            Err(err) => {
-                return Err(anyhow::anyhow!(
-                    "Failed to create Anthropic client: {}",
-                    err.to_string()
-                ));
-            }
-        };
+        let base_url = &config.base_url[..config.base_url.len() - 3];
+        let client = ClientBuilder::default()
+            .api_base(base_url.to_string())
+            .api_key(config.api_key.clone())
+            .build()?;
 
         Ok(Self {
             config,
@@ -71,7 +71,8 @@ impl ControllerTrait for Controller {
 
         let messages = vec![LLMMessage {
             author: LLMAuthor::User,
-            message_text: "Hello!".to_string(),
+            content: LLMMessageContent::Text("Hello!".to_string()),
+            timestamp: chrono::Utc::now(),
         }];
 
         let conversation = LLMConversation { messages };
@@ -95,21 +96,32 @@ impl ControllerTrait for Controller {
             ));
         };
 
-        let prompt_text = params
-            .prompt_override
-            .unwrap_or(self.text_generation_prompt().unwrap_or("".to_owned()))
-            .trim()
-            .to_owned();
+        let prompt_text = params.prompt_variables.format(
+            params
+                .prompt_override
+                .unwrap_or(self.text_generation_prompt().unwrap_or("".to_owned()))
+                .trim(),
+        );
 
         let prompt_message = if prompt_text.is_empty() {
             None
         } else {
             Some(LLMMessage {
                 author: LLMAuthor::Prompt,
-                message_text: prompt_text,
+                content: LLMMessageContent::Text(prompt_text),
+                timestamp: chrono::Utc::now(),
             })
         };
 
+        // Avoid the situation where multiple user or assistant messages are sent consecutively,
+        // to avoid errors like:
+        // > API error: Error response: error Api error: invalid_request_error messages: roles must alternate between "user" and "assistant", but found multiple "user" roles in a row
+        // as reported here: https://github.com/etkecc/baibot/issues/13
+        //
+        // As https://docs.anthropic.com/en/api/messages says:
+        // > Our models are trained to operate on alternating user and assistant conversational turns.
+        let conversation = conversation.combine_consecutive_messages();
+
         let mut conversation_messages = conversation.messages;
 
         if params.context_management_enabled {
@@ -119,7 +131,7 @@ impl ControllerTrait for Controller {
                 &text_generation_config.model_id,
                 &prompt_message,
                 conversation_messages,
-                text_generation_config.max_response_tokens,
+                Some(text_generation_config.max_response_tokens),
                 text_generation_config.max_context_tokens,
             );
 
@@ -130,29 +142,19 @@ impl ControllerTrait for Controller {
 
         let mut request = super::utils::create_anthropic_message_request(conversation_messages);
 
-        let model = match ClaudeModel::from_str(&text_generation_config.model_id) {
-            Ok(model) => model,
-            Err(err) => {
-                tracing::debug!(?err, "Failed to parse model ID");
-
-                return Err(anyhow::anyhow!(
-                    "Failed to parse model ID: {}",
-                    &text_generation_config.model_id
-                ));
-            }
-        };
-
         let temperature = params
             .temperature_override
             .unwrap_or(text_generation_config.temperature);
 
-        if let Some(prompt_message) = prompt_message {
-            request.system = Some(prompt_message.message_text);
+        if let Some(prompt_message) = prompt_message
+            && let LLMMessageContent::Text(text) = &prompt_message.content
+        {
+            request.system = text.clone();
         }
 
-        request.model = model;
-        request.temperature = Some(temperature);
-        request.max_tokens = text_generation_config.max_response_tokens;
+        request.model = text_generation_config.model_id.clone();
+        request.temperature = Some(temperature as f64);
+        request.max_tokens = text_generation_config.max_response_tokens as usize;
 
         if let Ok(request_as_json) = serde_json::to_string(&request) {
             tracing::trace!(
@@ -163,19 +165,20 @@ impl ControllerTrait for Controller {
             );
         }
 
-        let response = self.inner.client.create_message(request).await?;
+        let response = self.inner.client.messages(request).await?;
 
         tracing::trace!(?response, "Got response from Anthropic create message API");
 
         // response.content usually contains a single element, but we support handling multiple to account for all possibilities
         let mut text_parts = vec![];
         for content in response.content {
-            let content_type = content.content_type;
-
-            match content_type {
-                ContentType::Text => {
-                    text_parts.push(content.text);
-                } // There are no other content types to handle yet, but there may be in the future
+            match content {
+                ContentBlock::Text { text } => {
+                    text_parts.push(text);
+                }
+                ContentBlock::Image { .. } => {
+                    text_parts.push("The model responded with an image".to_string());
+                }
             }
         }
 
@@ -207,6 +210,15 @@ impl ControllerTrait for Controller {
         Err(anyhow::anyhow!("Image generation not supported"))
     }
 
+    async fn create_image_edit(
+        &self,
+        _prompt: &str,
+        _images: Vec<ImageSource>,
+        _params: ImageEditParams,
+    ) -> anyhow::Result<ImageEditResult> {
+        Err(anyhow::anyhow!("Image editing is not supported"))
+    }
+
     async fn text_to_speech(
         &self,
         _input: &str,
@@ -225,20 +237,25 @@ impl ControllerTrait for Controller {
         }
     }
 
-    fn text_generation_prompt(&self) -> Option<String> {
-        let Some(text_generation_config) = &self.config.text_generation else {
-            return None;
-        };
+    fn text_generation_model_id(&self) -> Option<String> {
+        self.config
+            .text_generation
+            .as_ref()
+            .map(|config| config.model_id.to_owned())
+    }
 
-        text_generation_config.prompt.clone()
+    fn text_generation_prompt(&self) -> Option<String> {
+        self.config
+            .text_generation
+            .as_ref()
+            .and_then(|config| config.prompt.clone())
     }
 
     fn text_generation_temperature(&self) -> Option<f32> {
-        let Some(text_generation_config) = &self.config.text_generation else {
-            return None;
-        };
-
-        Some(text_generation_config.temperature)
+        self.config
+            .text_generation
+            .as_ref()
+            .map(|config| config.temperature)
     }
 
     fn text_to_speech_voice(&self) -> Option<String> {

src/agent/provider/anthropic/mod.rs

@@ -7,17 +7,17 @@ pub use controller::Controller;
 
 use super::super::AgentInstantiationError;
 use super::super::AgentInstantiationResult;
-use super::controller::ControllerType;
 use super::ConfigTrait;
+use super::controller::ControllerType;
 
 pub fn create_controller_from_yaml_value_config(
     agent_id: &str,
-    config: serde_yaml::Value,
+    config: serde_yaml_ng::Value,
 ) -> AgentInstantiationResult<ControllerType> {
     let config = match &config {
-        serde_yaml::Value::Mapping(_) => {
+        serde_yaml_ng::Value::Mapping(_) => {
             let config: Config =
-                serde_yaml::from_value(config).map_err(AgentInstantiationError::Yaml)?;
+                serde_yaml_ng::from_value(config).map_err(AgentInstantiationError::Yaml)?;
 
             config
                 .validate()

src/agent/provider/anthropic/utils.rs

@@ -1,8 +1,12 @@
-use anthropic_rs::completion::message::{Content, ContentType, Message, MessageRequest, Role};
+use anthropic::types::{
+    ContentBlock, ImageSource, Message, MessagesRequest, MessagesRequestBuilder, Role,
+};
 
-use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage};
+use crate::conversation::llm::{
+    Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent,
+};
 
-pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) -> MessageRequest {
+pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) -> MessagesRequest {
     let mut messages = vec![];
 
     for message in llm_messages {
@@ -14,19 +18,26 @@ pub(super) fn create_anthropic_message_request(llm_messages: Vec<LLMMessage>) ->
             }
         };
 
-        let content = vec![Content {
-            content_type: ContentType::Text,
-            text: message.message_text,
-        }];
+        let content = match &message.content {
+            LLMMessageContent::Text(text) => vec![ContentBlock::Text { text: text.clone() }],
+            LLMMessageContent::Image(image_details) => {
+                vec![ContentBlock::Image {
+                    source: ImageSource::Base64 {
+                        media_type: image_details.mime.to_string(),
+                        data: crate::utils::base64::base64_encode(&image_details.data),
+                    },
+                }]
+            }
+        };
 
         let message = Message { role, content };
 
         messages.push(message);
     }
 
-    MessageRequest {
-        stream: false,
-        messages,
-        ..Default::default()
-    }
+    MessagesRequestBuilder::default()
+        .messages(messages)
+        .stream(false)
+        .build()
+        .expect("Failed to build messages request")
 }

src/agent/provider/controller.rs

@@ -1,11 +1,11 @@
 use crate::{agent::AgentPurpose, conversation::llm::Conversation};
 
 use super::{
+    ImageEditParams, ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
     entity::{
-        ImageGenerationResult, PingResult, TextGenerationParams, TextGenerationResult,
-        TextToSpeechParams, TextToSpeechResult,
+        ImageEditResult, ImageGenerationResult, ImageSource, PingResult, TextGenerationParams,
+        TextGenerationResult, TextToSpeechParams, TextToSpeechResult,
     },
-    ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
 };
 
 pub trait ControllerTrait {
@@ -13,6 +13,8 @@ pub trait ControllerTrait {
 
     fn ping(&self) -> impl std::future::Future<Output = anyhow::Result<PingResult>> + Send;
 
+    fn text_generation_model_id(&self) -> Option<String>;
+
     fn text_generation_prompt(&self) -> Option<String>;
 
     fn text_generation_temperature(&self) -> Option<f32>;
@@ -40,6 +42,13 @@ pub trait ControllerTrait {
         params: ImageGenerationParams,
     ) -> impl std::future::Future<Output = anyhow::Result<ImageGenerationResult>> + Send;
 
+    fn create_image_edit(
+        &self,
+        prompt: &str,
+        images: Vec<ImageSource>,
+        params: ImageEditParams,
+    ) -> impl std::future::Future<Output = anyhow::Result<ImageEditResult>> + Send;
+
     fn text_to_speech(
         &self,
         text: &str,
@@ -63,6 +72,14 @@ impl ControllerTrait for ControllerType {
         }
     }
 
+    fn text_generation_model_id(&self) -> Option<String> {
+        match &self {
+            ControllerType::OpenAI(controller) => controller.text_generation_model_id(),
+            ControllerType::OpenAICompat(controller) => controller.text_generation_model_id(),
+            ControllerType::Anthropic(controller) => controller.text_generation_model_id(),
+        }
+    }
+
     fn text_generation_prompt(&self) -> Option<String> {
         match &self {
             ControllerType::OpenAI(controller) => controller.text_generation_prompt(),
@@ -156,6 +173,25 @@ impl ControllerTrait for ControllerType {
         }
     }
 
+    async fn create_image_edit(
+        &self,
+        prompt: &str,
+        images: Vec<ImageSource>,
+        params: ImageEditParams,
+    ) -> anyhow::Result<ImageEditResult> {
+        match &self {
+            ControllerType::OpenAI(controller) => {
+                controller.create_image_edit(prompt, images, params).await
+            }
+            ControllerType::OpenAICompat(controller) => {
+                controller.create_image_edit(prompt, images, params).await
+            }
+            ControllerType::Anthropic(controller) => {
+                controller.create_image_edit(prompt, images, params).await
+            }
+        }
+    }
+
     async fn text_to_speech(
         &self,
         text: &str,

src/agent/provider/entity/agent_provider.rs

@@ -67,9 +67,9 @@ impl AgentProvider {
                 wiki_url: Some("https://en.wikipedia.org/wiki/Anthropic"),
                 sign_up_url: Some("https://console.anthropic.com/"),
                 models_list_url: Some("https://docs.anthropic.com/en/docs/about-claude/models"),
-                supported_purposes: vec![
-                    AgentPurpose::TextGeneration,
-                ],
+                supported_purposes: vec![AgentPurpose::TextGeneration],
+                text_generation_supports_vision: true,
+                text_generation_supports_tools: false,
             },
             Self::Groq => AgentProviderInfo {
                 id: Self::Groq.to_static_str(),
@@ -79,15 +79,14 @@ impl AgentProvider {
                 wiki_url: Some("https://en.wikipedia.org/wiki/Groq"),
                 sign_up_url: Some("https://console.groq.com/login"),
                 models_list_url: Some("https://console.groq.com/docs/models"),
-                supported_purposes: vec![
-                    AgentPurpose::TextGeneration,
-                    AgentPurpose::SpeechToText,
-                ],
+                supported_purposes: vec![AgentPurpose::TextGeneration, AgentPurpose::SpeechToText],
+                text_generation_supports_vision: false,
+                text_generation_supports_tools: false,
             },
             Self::LocalAI => AgentProviderInfo {
                 id: Self::LocalAI.to_static_str(),
                 name: "LocalAI",
-                description: "LocalAI is the free, Open Source OpenAI alternative. LocalAI act as a drop-in replacement REST API that’s compatible with OpenAI API specifications for local inferencing. It allows you to run LLMs, generate images, audio (and not only) locally or on-prem with consumer grade hardware, supporting multiple model families and architectures.",
+                description: "LocalAI is the free, Open Source OpenAI alternative. LocalAI act as a drop-in replacement REST API that's compatible with OpenAI API specifications for local inferencing. It allows you to run LLMs, generate images, audio (and not only) locally or on-prem with consumer grade hardware, supporting multiple model families and architectures.",
                 homepage_url: Some("https://localai.io/"),
                 wiki_url: None,
                 sign_up_url: None,
@@ -97,6 +96,8 @@ impl AgentProvider {
                     AgentPurpose::TextToSpeech,
                     AgentPurpose::SpeechToText,
                 ],
+                text_generation_supports_vision: false,
+                text_generation_supports_tools: false,
             },
             Self::Mistral => AgentProviderInfo {
                 id: Self::Mistral.to_static_str(),
@@ -106,9 +107,9 @@ impl AgentProvider {
                 wiki_url: Some("https://en.wikipedia.org/wiki/Mistral_AI"),
                 sign_up_url: Some("https://auth.mistral.ai/ui/registration"),
                 models_list_url: Some("https://docs.mistral.ai/getting-started/models/"),
-                supported_purposes: vec![
-                    AgentPurpose::TextGeneration,
-                ],
+                supported_purposes: vec![AgentPurpose::TextGeneration],
+                text_generation_supports_vision: false,
+                text_generation_supports_tools: false,
             },
             Self::Ollama => AgentProviderInfo {
                 id: Self::Ollama.to_static_str(),
@@ -118,9 +119,9 @@ impl AgentProvider {
                 wiki_url: None,
                 sign_up_url: None,
                 models_list_url: Some("https://ollama.com/library"),
-                supported_purposes: vec![
-                    AgentPurpose::TextGeneration,
-                ],
+                supported_purposes: vec![AgentPurpose::TextGeneration],
+                text_generation_supports_vision: false,
+                text_generation_supports_tools: false,
             },
             Self::OpenAI => AgentProviderInfo {
                 id: Self::OpenAI.to_static_str(),
@@ -136,6 +137,8 @@ impl AgentProvider {
                     AgentPurpose::TextToSpeech,
                     AgentPurpose::SpeechToText,
                 ],
+                text_generation_supports_vision: true,
+                text_generation_supports_tools: true,
             },
             Self::OpenAICompat => AgentProviderInfo {
                 id: Self::OpenAICompat.to_static_str(),
@@ -151,6 +154,8 @@ impl AgentProvider {
                     AgentPurpose::TextToSpeech,
                     AgentPurpose::SpeechToText,
                 ],
+                text_generation_supports_vision: false,
+                text_generation_supports_tools: false,
             },
             Self::OpenRouter => AgentProviderInfo {
                 id: Self::OpenRouter.to_static_str(),
@@ -160,9 +165,9 @@ impl AgentProvider {
                 wiki_url: None,
                 sign_up_url: Some("https://openrouter.ai/"),
                 models_list_url: Some("https://openrouter.ai/models"),
-                supported_purposes: vec![
-                    AgentPurpose::TextGeneration,
-                ],
+                supported_purposes: vec![AgentPurpose::TextGeneration],
+                text_generation_supports_vision: false,
+                text_generation_supports_tools: false,
             },
             Self::TogetherAI => AgentProviderInfo {
                 id: Self::TogetherAI.to_static_str(),
@@ -172,9 +177,9 @@ impl AgentProvider {
                 wiki_url: None,
                 sign_up_url: Some("https://api.together.ai/signup"),
                 models_list_url: Some("https://api.together.xyz/models"),
-                supported_purposes: vec![
-                    AgentPurpose::TextGeneration,
-                ],
+                supported_purposes: vec![AgentPurpose::TextGeneration],
+                text_generation_supports_vision: false,
+                text_generation_supports_tools: false,
             },
         }
     }
@@ -195,4 +200,6 @@ pub struct AgentProviderInfo {
     pub sign_up_url: Option<&'static str>,
     pub models_list_url: Option<&'static str>,
     pub supported_purposes: Vec<AgentPurpose>,
+    pub text_generation_supports_vision: bool,
+    pub text_generation_supports_tools: bool,
 }

src/agent/provider/entity/image.rs

@@ -0,0 +1,63 @@
+use mxlink::mime;
+
+#[derive(Default)]
+pub struct ImageGenerationParams {
+    pub smallest_size_possible: bool,
+
+    pub cheaper_model_switching_allowed: bool,
+
+    pub cheaper_quality_switching_allowed: bool,
+}
+
+impl ImageGenerationParams {
+    pub fn with_smallest_size_possible(mut self, value: bool) -> Self {
+        self.smallest_size_possible = value;
+        self
+    }
+
+    pub fn with_cheaper_model_switching_allowed(mut self, value: bool) -> Self {
+        self.cheaper_model_switching_allowed = value;
+        self
+    }
+
+    pub fn with_cheaper_quality_switching_allowed(mut self, value: bool) -> Self {
+        self.cheaper_quality_switching_allowed = value;
+        self
+    }
+}
+
+pub struct ImageGenerationResult {
+    pub bytes: Vec<u8>,
+    pub mime_type: mime::Mime,
+    pub revised_prompt: Option<String>,
+}
+
+#[derive(Default)]
+pub struct ImageEditParams {}
+
+pub struct ImageEditResult {
+    pub bytes: Vec<u8>,
+    pub mime_type: mime::Mime,
+}
+
+pub struct ImageSource {
+    pub filename: String,
+    pub bytes: Vec<u8>,
+    pub mime_type: mime::Mime,
+}
+
+impl ImageSource {
+    pub fn new(filename: String, bytes: Vec<u8>, mime_type: mime::Mime) -> Self {
+        Self {
+            filename,
+            bytes,
+            mime_type,
+        }
+    }
+}
+
+impl From<ImageSource> for async_openai::types::images::ImageInput {
+    fn from(value: ImageSource) -> Self {
+        async_openai::types::images::ImageInput::from_vec_u8(value.filename, value.bytes)
+    }
+}

src/agent/provider/entity/image_generation.rs

@@ -1,31 +0,0 @@
-#[derive(Default)]
-pub struct ImageGenerationParams {
-    pub size_override: Option<String>,
-
-    pub cheaper_model_switching_allowed: bool,
-
-    pub cheaper_quality_switching_allowed: bool,
-}
-
-impl ImageGenerationParams {
-    pub fn with_size_override(mut self, value: Option<String>) -> Self {
-        self.size_override = value;
-        self
-    }
-
-    pub fn with_cheaper_model_switching_allowed(mut self, value: bool) -> Self {
-        self.cheaper_model_switching_allowed = value;
-        self
-    }
-
-    pub fn with_cheaper_quality_switching_allowed(mut self, value: bool) -> Self {
-        self.cheaper_quality_switching_allowed = value;
-        self
-    }
-}
-
-pub struct ImageGenerationResult {
-    pub bytes: Vec<u8>,
-    pub mime_type: mxlink::mime::Mime,
-    pub revised_prompt: Option<String>,
-}

src/agent/provider/entity/mod.rs

@@ -1,13 +1,17 @@
 mod agent_provider;
-mod image_generation;
+mod image;
 mod ping;
 mod speech_to_text;
 mod text_generation;
 mod text_to_speech;
 
 pub use agent_provider::{AgentProvider, AgentProviderInfo};
-pub use image_generation::{ImageGenerationParams, ImageGenerationResult};
+pub use image::{
+    ImageEditParams, ImageEditResult, ImageGenerationParams, ImageGenerationResult, ImageSource,
+};
 pub use ping::PingResult;
 pub use speech_to_text::{SpeechToTextParams, SpeechToTextResult};
-pub use text_generation::{TextGenerationParams, TextGenerationResult};
+pub use text_generation::{
+    TextGenerationParams, TextGenerationPromptVariables, TextGenerationResult,
+};
 pub use text_to_speech::{TextToSpeechParams, TextToSpeechResult};

src/agent/provider/entity/text_generation/mod.rs

@@ -1,8 +1,13 @@
+mod prompt_variables;
+
+pub use prompt_variables::TextGenerationPromptVariables;
+
 #[derive(Default)]
 pub struct TextGenerationParams {
     pub context_management_enabled: bool,
     pub prompt_override: Option<String>,
     pub temperature_override: Option<f32>,
+    pub prompt_variables: TextGenerationPromptVariables,
 }
 
 pub struct TextGenerationResult {

src/agent/provider/entity/text_generation/prompt_variables.rs

@@ -0,0 +1,106 @@
+use chrono::{DateTime, Utc};
+use std::collections::HashMap;
+
+pub struct TextGenerationPromptVariables {
+    map: HashMap<String, String>,
+}
+
+impl Default for TextGenerationPromptVariables {
+    fn default() -> Self {
+        let now = Utc::now();
+        Self::new("unnamed", "unknown-model", now, Some(now))
+    }
+}
+
+impl TextGenerationPromptVariables {
+    pub fn new(
+        bot_name: &str,
+        model_id: &str,
+        now_time: DateTime<Utc>,
+        conversation_start_time: Option<DateTime<Utc>>,
+    ) -> Self {
+        let mut map = HashMap::new();
+
+        map.insert("baibot_name".to_string(), bot_name.to_string());
+        map.insert("baibot_model_id".to_string(), model_id.to_string());
+        map.insert("baibot_now_utc".to_string(), format_utc_time(now_time));
+
+        let baibot_conversation_start_time_utc = match conversation_start_time {
+            Some(conversation_start_time) => format_utc_time(conversation_start_time),
+            None => "unknown".to_string(),
+        };
+
+        map.insert(
+            "baibot_conversation_start_time_utc".to_string(),
+            baibot_conversation_start_time_utc,
+        );
+
+        Self { map }
+    }
+
+    pub fn format(&self, text: &str) -> String {
+        let mut formatted_text = text.to_string();
+
+        for (key, value) in &self.map {
+            let placeholder = format!("{{{{ {} }}}}", key);
+            formatted_text = formatted_text.replace(&placeholder, value);
+        }
+
+        formatted_text
+    }
+}
+
+fn format_utc_time(time: DateTime<Utc>) -> String {
+    time.format("%Y-%m-%d (%A), %H:%M:%S UTC").to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use chrono::{TimeZone, Timelike};
+
+    #[test]
+    fn test_new() {
+        // Intentionally injecting some sub-seconds to ensure formatting would ignore them.
+        let now_utc = Utc
+            .with_ymd_and_hms(2024, 9, 20, 18, 34, 15)
+            .unwrap()
+            .with_nanosecond(250000000)
+            .unwrap();
+
+        let conversation_start_time_utc = Utc
+            .with_ymd_and_hms(2024, 9, 19, 18, 34, 15)
+            .unwrap()
+            .with_nanosecond(250000000)
+            .unwrap();
+
+        let variables = TextGenerationPromptVariables::new(
+            "baibot",
+            "gpt-4o",
+            now_utc,
+            Some(conversation_start_time_utc),
+        );
+
+        assert_eq!(
+            variables.map.get("baibot_name"),
+            Some(&"baibot".to_string())
+        );
+        assert_eq!(
+            variables.map.get("baibot_model_id"),
+            Some(&"gpt-4o".to_string())
+        );
+        assert_eq!(
+            variables.map.get("baibot_now_utc"),
+            Some(&format_utc_time(now_utc))
+        );
+        assert_eq!(
+            variables.map.get("baibot_conversation_start_time_utc"),
+            Some(&format_utc_time(conversation_start_time_utc))
+        );
+
+        let prompt = "Hello, I'm {{ baibot_name }} using {{ baibot_model_id }}. The date/time now is {{ baibot_now_utc }} and this conversation started at {{ baibot_conversation_start_time_utc }}.";
+        let expected = "Hello, I'm baibot using gpt-4o. The date/time now is 2024-09-20 (Friday), 18:34:15 UTC and this conversation started at 2024-09-19 (Thursday), 18:34:15 UTC.";
+
+        assert_eq!(variables.format(prompt), expected);
+    }
+}

src/agent/provider/groq/mod.rs

@@ -15,7 +15,7 @@ pub fn default_config() -> Config {
     if let Some(ref mut config) = config.text_generation.as_mut() {
         config.model_id = "llama3-70b-8192".to_owned();
         config.max_context_tokens = 131_072;
-        config.max_response_tokens = 4096;
+        config.max_response_tokens = Some(4096);
     }
 
     if let Some(ref mut config) = config.speech_to_text.as_mut() {

src/agent/provider/localai/mod.rs

@@ -1,6 +1,5 @@
-// LocalAI is based on OpenAI (async-openai), because it seems to be fully compatible.
-// Moreover, openai_api_rust does not support speech-to-text, so if we wish to use this feature
-// we need to stick to async-openai.
+// At the time of testing, LocalAI can be powered by `openai`, but we use `openai_compat` for better reliability
+// in the event of future updates to `async-openai`.
 
 use super::openai_compat::Config;
 
@@ -14,7 +13,7 @@ pub fn default_config() -> Config {
     if let Some(ref mut config) = config.text_generation.as_mut() {
         config.model_id = "gpt-4".to_owned();
         config.max_context_tokens = 128_000;
-        config.max_response_tokens = 4096;
+        config.max_response_tokens = Some(4096);
     }
 
     if let Some(ref mut config) = config.text_to_speech.as_mut() {

src/agent/provider/mod.rs

@@ -20,6 +20,7 @@ pub use controller::{ControllerTrait, ControllerType};
 pub use config::ConfigTrait;
 
 pub use entity::{
-    AgentProvider, AgentProviderInfo, ImageGenerationParams, PingResult, SpeechToTextParams,
-    SpeechToTextResult, TextGenerationParams, TextToSpeechParams,
+    AgentProvider, AgentProviderInfo, ImageEditParams, ImageGenerationParams, ImageSource,
+    PingResult, SpeechToTextParams, SpeechToTextResult, TextGenerationParams,
+    TextGenerationPromptVariables, TextToSpeechParams,
 };

src/agent/provider/ollama/mod.rs

@@ -17,7 +17,7 @@ pub fn default_config() -> Config {
     if let Some(ref mut config) = config.text_generation.as_mut() {
         config.model_id = "gemma2:2b".to_owned();
         config.max_context_tokens = 128_000;
-        config.max_response_tokens = 4096;
+        config.max_response_tokens = Some(4096);
     }
 
     config

src/agent/provider/openai/config.rs

@@ -1,6 +1,7 @@
 use serde::{Deserialize, Serialize};
 
-use crate::agent::provider::ConfigTrait;
+use super::OPENAI_IMAGE_MODEL_GPT_IMAGE_1_DOT_5;
+use crate::agent::{default_prompt, provider::ConfigTrait};
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct Config {
@@ -56,26 +57,43 @@ pub struct TextGenerationConfig {
     pub temperature: f32,
 
     #[serde(default)]
-    pub max_response_tokens: u32,
+    pub max_response_tokens: Option<u32>,
+
+    #[serde(default)]
+    pub max_completion_tokens: Option<u32>,
 
     #[serde(default)]
     pub max_context_tokens: u32,
+
+    #[serde(default)]
+    pub tools: ToolsConfig,
 }
 
 impl Default for TextGenerationConfig {
     fn default() -> Self {
         Self {
             model_id: default_text_model_id(),
-            prompt: Some("You are a brief, but helpful bot.".to_owned()),
+            prompt: Some(default_prompt().to_owned()),
             temperature: super::super::default_temperature(),
-            max_response_tokens: 16_384,
-            max_context_tokens: 128_000,
+            max_response_tokens: None,
+            max_completion_tokens: Some(128_000),
+            max_context_tokens: 400_000,
+            tools: ToolsConfig::default(),
         }
     }
 }
 
 fn default_text_model_id() -> String {
-    "gpt-4o-2024-08-06".to_owned()
+    "gpt-5.2".to_owned()
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct ToolsConfig {
+    #[serde(default)]
+    pub web_search: bool,
+
+    #[serde(default)]
+    pub code_interpreter: bool,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -99,16 +117,16 @@ fn default_speech_to_text_model_id() -> String {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct TextToSpeechConfig {
     #[serde(default = "default_text_to_speech_model_id")]
-    pub model_id: async_openai::types::SpeechModel,
+    pub model_id: async_openai::types::audio::SpeechModel,
 
     #[serde(default = "default_text_to_speech_voice")]
-    pub voice: async_openai::types::Voice,
+    pub voice: async_openai::types::audio::Voice,
 
     #[serde(default = "default_text_to_speech_speed")]
     pub speed: f32,
 
     #[serde(default = "default_text_to_speech_response_format")]
-    pub response_format: async_openai::types::SpeechResponseFormat,
+    pub response_format: async_openai::types::audio::SpeechResponseFormat,
 }
 
 impl Default for TextToSpeechConfig {
@@ -122,22 +140,22 @@ impl Default for TextToSpeechConfig {
     }
 }
 
-fn default_text_to_speech_model_id() -> async_openai::types::SpeechModel {
-    async_openai::types::SpeechModel::Tts1Hd
+fn default_text_to_speech_model_id() -> async_openai::types::audio::SpeechModel {
+    async_openai::types::audio::SpeechModel::Tts1Hd
 }
 
-fn default_text_to_speech_voice() -> async_openai::types::Voice {
-    async_openai::types::Voice::Onyx
+fn default_text_to_speech_voice() -> async_openai::types::audio::Voice {
+    async_openai::types::audio::Voice::Onyx
 }
 
 fn default_text_to_speech_speed() -> f32 {
     1.0
 }
 
-fn default_text_to_speech_response_format() -> async_openai::types::SpeechResponseFormat {
+fn default_text_to_speech_response_format() -> async_openai::types::audio::SpeechResponseFormat {
     // The API defaults to mp3, but we prefer Opus because it's smaller.
     // Our clients should all have support for it.
-    async_openai::types::SpeechResponseFormat::Opus
+    async_openai::types::audio::SpeechResponseFormat::Opus
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -145,19 +163,19 @@ pub struct ImageGenerationConfig {
     pub model_id: String,
 
     #[serde(default = "default_image_style")]
-    pub style: async_openai::types::ImageStyle,
+    pub style: Option<async_openai::types::images::ImageStyle>,
 
     #[serde(default = "default_image_size")]
-    pub size: async_openai::types::ImageSize,
+    pub size: Option<async_openai::types::images::ImageSize>,
 
     #[serde(default = "default_image_quality")]
-    pub quality: async_openai::types::ImageQuality,
+    pub quality: Option<async_openai::types::images::ImageQuality>,
 }
 
 impl Default for ImageGenerationConfig {
     fn default() -> Self {
         Self {
-            model_id: "dall-e-3".to_owned(),
+            model_id: OPENAI_IMAGE_MODEL_GPT_IMAGE_1_DOT_5.to_owned(),
             style: default_image_style(),
             size: default_image_size(),
             quality: default_image_quality(),
@@ -168,23 +186,28 @@ impl Default for ImageGenerationConfig {
 impl ImageGenerationConfig {
     pub fn model_id_as_openai_image_model(
         &self,
-    ) -> Result<async_openai::types::ImageModel, String> {
+    ) -> Result<async_openai::types::images::ImageModel, String> {
         match self.model_id.as_str() {
-            "dall-e-2" => Ok(async_openai::types::ImageModel::DallE2),
-            "dall-e-3" => Ok(async_openai::types::ImageModel::DallE3),
-            other => Ok(async_openai::types::ImageModel::Other(other.to_owned())),
+            "dall-e-2" => Ok(async_openai::types::images::ImageModel::DallE2),
+            "dall-e-3" => Ok(async_openai::types::images::ImageModel::DallE3),
+            "gpt-image-1" => Ok(async_openai::types::images::ImageModel::GptImage1),
+            "gpt-image-1.5" => Ok(async_openai::types::images::ImageModel::GptImage1dot5),
+            "gpt-image-1-mini" => Ok(async_openai::types::images::ImageModel::GptImage1Mini),
+            other => Ok(async_openai::types::images::ImageModel::Other(
+                other.to_owned(),
+            )),
         }
     }
 }
 
-fn default_image_style() -> async_openai::types::ImageStyle {
-    async_openai::types::ImageStyle::Vivid
+fn default_image_style() -> Option<async_openai::types::images::ImageStyle> {
+    None
 }
 
-fn default_image_size() -> async_openai::types::ImageSize {
-    async_openai::types::ImageSize::S1024x1024
+fn default_image_size() -> Option<async_openai::types::images::ImageSize> {
+    None
 }
 
-fn default_image_quality() -> async_openai::types::ImageQuality {
-    async_openai::types::ImageQuality::Standard
+fn default_image_quality() -> Option<async_openai::types::images::ImageQuality> {
+    None
 }

src/agent/provider/openai/controller.rs

@@ -1,37 +1,42 @@
 use std::ops::Deref;
 
 use async_openai::{
+    Client as OpenAIClient,
     config::OpenAIConfig,
     types::{
-        ChatCompletionRequestMessage, CreateChatCompletionRequestArgs, CreateImageRequestArgs,
-        CreateSpeechRequestArgs, CreateTranscriptionRequestArgs,
+        audio::{AudioInput, CreateSpeechRequestArgs, CreateTranscriptionRequestArgs},
+        images::{
+            CreateImageEditRequestArgs, CreateImageRequestArgs, Image, ImageInput, ImageModel,
+            ImageResponseFormat,
+        },
+        responses::{
+            CodeInterpreterContainerAuto, CodeInterpreterTool, CodeInterpreterToolContainer,
+            CreateResponseArgs, OutputItem, OutputMessageContent, Tool, WebSearchTool,
+        },
     },
-    Client as OpenAIClient,
 };
 
 use super::super::ControllerTrait;
 use crate::{
-    agent::{
-        provider::{
-            entity::{ImageGenerationResult, PingResult, TextToSpeechParams, TextToSpeechResult},
-            openai::utils::convert_string_to_enum,
-        },
-        AgentPurpose,
+    agent::provider::{
+        ImageEditParams, ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
+        entity::{TextGenerationParams, TextGenerationResult},
     },
-    strings,
+    conversation::llm::{
+        Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
+        MessageContent as LLMMessageContent, shorten_messages_list_to_context_size,
+    },
+    utils::base64::base64_decode,
 };
 use crate::{
     agent::{
-        provider::{
-            entity::{TextGenerationParams, TextGenerationResult},
-            ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
+        AgentPurpose,
+        provider::entity::{
+            ImageEditResult, ImageGenerationResult, ImageSource, PingResult, TextToSpeechParams,
+            TextToSpeechResult,
         },
-        utils::base64_decode,
-    },
-    conversation::llm::{
-        shorten_messages_list_to_context_size, Author as LLMAuthor,
-        Conversation as LLMConversation, Message as LLMMessage,
     },
+    strings,
 };
 
 use super::config::Config;
@@ -62,7 +67,8 @@ impl ControllerTrait for Controller {
 
         let messages = vec![LLMMessage {
             author: LLMAuthor::User,
-            message_text: "Hello!".to_string(),
+            content: LLMMessageContent::Text("Hello!".to_string()),
+            timestamp: chrono::Utc::now(),
         }];
 
         let conversation = LLMConversation { messages };
@@ -86,18 +92,20 @@ impl ControllerTrait for Controller {
             ));
         };
 
-        let prompt_text = params
-            .prompt_override
-            .unwrap_or(self.text_generation_prompt().unwrap_or("".to_owned()))
-            .trim()
-            .to_owned();
+        let prompt_text = params.prompt_variables.format(
+            params
+                .prompt_override
+                .unwrap_or(self.text_generation_prompt().unwrap_or("".to_owned()))
+                .trim(),
+        );
 
         let prompt_message = if prompt_text.is_empty() {
             None
         } else {
             Some(LLMMessage {
                 author: LLMAuthor::Prompt,
-                message_text: prompt_text,
+                content: LLMMessageContent::Text(prompt_text),
+                timestamp: chrono::Utc::now(),
             })
         };
 
@@ -121,54 +129,76 @@ impl ControllerTrait for Controller {
             conversation_messages.insert(0, prompt_message);
         }
 
-        let openai_conversation_messages: Vec<ChatCompletionRequestMessage> =
-            super::utils::convert_llm_messages_to_openai_messages(conversation_messages);
+        let input =
+            super::utils::convert_llm_messages_to_openai_response_input(conversation_messages);
 
-        let messages_count = openai_conversation_messages.len();
+        let messages_count = match &input {
+            async_openai::types::responses::InputParam::Items(items) => items.len(),
+            _ => 1,
+        };
 
         let temperature = params
             .temperature_override
             .unwrap_or(text_generation_config.temperature);
 
-        let request = CreateChatCompletionRequestArgs::default()
-            .max_tokens(text_generation_config.max_response_tokens)
+        let mut request_builder = CreateResponseArgs::default();
+
+        request_builder
             .model(&text_generation_config.model_id)
             .temperature(temperature)
-            .messages(openai_conversation_messages)
-            .build()?;
+            .input(input);
+
+        let mut tools = Vec::new();
+        if text_generation_config.tools.web_search {
+            tools.push(Tool::WebSearch(WebSearchTool::default()));
+        }
+        if text_generation_config.tools.code_interpreter {
+            tools.push(Tool::CodeInterpreter(CodeInterpreterTool {
+                container: CodeInterpreterToolContainer::Auto(
+                    CodeInterpreterContainerAuto::default(),
+                ),
+            }));
+        }
+
+        if !tools.is_empty() {
+            request_builder.tools(tools);
+        }
+
+        if let Some(max_response_tokens) = text_generation_config.max_response_tokens {
+            request_builder.max_output_tokens(max_response_tokens);
+        } else if let Some(max_completion_tokens) = text_generation_config.max_completion_tokens {
+            request_builder.max_output_tokens(max_completion_tokens);
+        }
+
+        let request = request_builder.build()?;
 
         if let Ok(request_as_json) = serde_json::to_string(&request) {
             tracing::trace!(
                 model = format!("{:?}", request.model),
                 ?messages_count,
                 request = request_as_json,
-                "Sending OpenAI chat completion API request"
+                "Sending OpenAI response API request"
             );
         }
 
-        let response = self.client.chat().create(request).await?;
+        let response = self.client.responses().create(request).await?;
 
-        tracing::trace!(
-            ?response,
-            "Got response from the OpenAI chat completion API"
-        );
+        tracing::trace!(?response, "Got response from the OpenAI response API");
 
-        // We only request 1 result, so there should only be 1 choice.
-        if let Some(choice) = response.choices.into_iter().next() {
-            match choice.message.content {
-                Some(text) => {
-                    return Ok(TextGenerationResult { text });
-                }
-                None => {
-                    return Err(anyhow::anyhow!(
-                        "No content was found in the response choice from the OpenAI chat completion API"
-                    ));
+        for item in response.output {
+            if let OutputItem::Message(message) = item {
+                for content in message.content {
+                    if let OutputMessageContent::OutputText(text_content) = content {
+                        return Ok(TextGenerationResult {
+                            text: text_content.text,
+                        });
+                    }
                 }
             }
         }
 
         Err(anyhow::anyhow!(
-            "No response messages choices were returned from the OpenAI chat completion API"
+            "No response messages choices were returned from the OpenAI response API"
         ))
     }
 
@@ -192,12 +222,7 @@ impl ControllerTrait for Controller {
 
         let request = CreateTranscriptionRequestArgs::default()
             .model(&speech_to_text_config.model_id)
-            .file(async_openai::types::AudioInput {
-                source: async_openai::types::InputSource::VecU8 {
-                    filename,
-                    vec: media,
-                },
-            })
+            .file(AudioInput::from_vec_u8(filename, media))
             .language(language.clone())
             .build()?;
 
@@ -207,7 +232,7 @@ impl ControllerTrait for Controller {
             "Sending OpenAI speech-to-text API request"
         );
 
-        let response = self.client.audio().transcribe(request).await?;
+        let response = self.client.audio().transcription().create(request).await?;
 
         tracing::trace!(
             ?response,
@@ -239,11 +264,12 @@ impl ControllerTrait for Controller {
         let model = if params.cheaper_model_switching_allowed {
             // Switch to a cheaper model
             match original_model {
-                async_openai::types::ImageModel::DallE2 => async_openai::types::ImageModel::DallE2,
-                async_openai::types::ImageModel::DallE3 => async_openai::types::ImageModel::DallE2,
-                async_openai::types::ImageModel::Other(_) => {
-                    async_openai::types::ImageModel::DallE2
-                }
+                ImageModel::DallE2 => ImageModel::DallE2,
+                ImageModel::DallE3 => ImageModel::DallE2,
+                ImageModel::GptImage1 => ImageModel::GptImage1Mini,
+                ImageModel::GptImage1dot5 => ImageModel::GptImage1Mini,
+                ImageModel::GptImage1Mini => ImageModel::GptImage1Mini,
+                ImageModel::Other(_) => ImageModel::DallE2,
             }
         } else {
             original_model
@@ -252,33 +278,71 @@ impl ControllerTrait for Controller {
         let quality = if params.cheaper_quality_switching_allowed {
             // Switch to a cheaper quality
             match &image_generation_config.quality {
-                async_openai::types::ImageQuality::Standard => {
-                    async_openai::types::ImageQuality::Standard
-                }
-                async_openai::types::ImageQuality::HD => {
-                    async_openai::types::ImageQuality::Standard
-                }
+                Some(quality) => match quality {
+                    async_openai::types::images::ImageQuality::Standard => {
+                        Some(async_openai::types::images::ImageQuality::Standard)
+                    }
+                    async_openai::types::images::ImageQuality::HD => {
+                        Some(async_openai::types::images::ImageQuality::Standard)
+                    }
+                    // New quality levels - keep as-is or downgrade to Standard
+                    async_openai::types::images::ImageQuality::High => {
+                        Some(async_openai::types::images::ImageQuality::Standard)
+                    }
+                    async_openai::types::images::ImageQuality::Medium => {
+                        Some(async_openai::types::images::ImageQuality::Medium)
+                    }
+                    async_openai::types::images::ImageQuality::Low => {
+                        Some(async_openai::types::images::ImageQuality::Low)
+                    }
+                    async_openai::types::images::ImageQuality::Auto => {
+                        Some(async_openai::types::images::ImageQuality::Auto)
+                    }
+                },
+                None => None,
             }
         } else {
             image_generation_config.quality.clone()
         };
 
-        let size = params
-            .size_override
-            .map(|s| {
-                convert_string_to_enum::<async_openai::types::ImageSize>(&s)
-                    .unwrap_or(image_generation_config.size)
-            })
-            .unwrap_or(image_generation_config.size);
+        let size = if params.smallest_size_possible {
+            Some(get_sticker_size(&model))
+        } else {
+            image_generation_config.size
+        };
 
-        let request = CreateImageRequestArgs::default()
-            .model(model)
-            .prompt(prompt.to_owned())
-            .response_format(async_openai::types::ImageResponseFormat::B64Json)
-            .size(size)
-            .style(image_generation_config.style.clone())
-            .quality(quality)
-            .build()?;
+        let response_format = match model.clone() {
+            ImageModel::DallE2 => Some(ImageResponseFormat::B64Json),
+            ImageModel::DallE3 => Some(ImageResponseFormat::B64Json),
+            // gpt-image-1 only outputs base64 and we don't need to specify the response format.
+            // In fact, specifying the response format results in an error.
+            ImageModel::GptImage1 => None,
+            ImageModel::GptImage1Mini => None,
+            ImageModel::GptImage1dot5 => None,
+            ImageModel::Other(_) => Some(ImageResponseFormat::B64Json),
+        };
+
+        let mut request_builder = CreateImageRequestArgs::default();
+
+        request_builder.model(model).prompt(prompt.to_owned());
+
+        if let Some(response_format) = response_format {
+            request_builder.response_format(response_format);
+        }
+
+        if let Some(style) = &image_generation_config.style {
+            request_builder.style(style.clone());
+        }
+
+        if let Some(quality) = quality {
+            request_builder.quality(quality.clone());
+        }
+
+        if let Some(size) = size {
+            request_builder.size(size);
+        }
+
+        let request = request_builder.build()?;
 
         tracing::trace!(
             ?prompt,
@@ -289,15 +353,15 @@ impl ControllerTrait for Controller {
             "Sending OpenAI image generation API request"
         );
 
-        let response = self.client.images().create(request).await?;
+        let response = self.client.images().generate(request).await?;
 
         if let Some(image) = response.data.into_iter().next() {
             match image.deref() {
-                async_openai::types::Image::B64Json {
+                Image::B64Json {
                     b64_json,
                     revised_prompt,
                 } => {
-                    let bytes = base64_decode(b64_json)?;
+                    let bytes = base64_decode(b64_json.as_ref())?;
 
                     return Ok(ImageGenerationResult {
                         bytes,
@@ -316,6 +380,108 @@ impl ControllerTrait for Controller {
         ))
     }
 
+    async fn create_image_edit(
+        &self,
+        prompt: &str,
+        images: Vec<ImageSource>,
+        _params: ImageEditParams,
+    ) -> anyhow::Result<ImageEditResult> {
+        let Some(image_generation_config) = &self.config.image_generation else {
+            return Err(anyhow::anyhow!(
+                strings::agent::no_configuration_for_purpose_so_cannot_be_used(
+                    &AgentPurpose::ImageGeneration
+                ),
+            ));
+        };
+
+        if images.is_empty() {
+            return Err(anyhow::anyhow!("No image sources provided"));
+        }
+
+        let mut image_inputs: Vec<ImageInput> = Vec::new();
+        for image in images {
+            image_inputs.push(image.into());
+        }
+
+        let dalle2_size = match image_generation_config.size {
+            Some(async_openai::types::images::ImageSize::S256x256) => {
+                Some(async_openai::types::images::ImageSize::S256x256)
+            }
+            Some(async_openai::types::images::ImageSize::S512x512) => {
+                Some(async_openai::types::images::ImageSize::S512x512)
+            }
+            Some(async_openai::types::images::ImageSize::S1024x1024) => {
+                Some(async_openai::types::images::ImageSize::S1024x1024)
+            }
+            _ => None,
+        };
+
+        let model = image_generation_config
+            .model_id_as_openai_image_model()
+            .map_err(|err| anyhow::anyhow!(err))?;
+
+        let response_format = match model.clone() {
+            ImageModel::DallE2 => Some(ImageResponseFormat::B64Json),
+            ImageModel::DallE3 => Some(ImageResponseFormat::B64Json),
+            // gpt-image-1 only outputs base64 and we don't need to specify the response format.
+            // In fact, specifying the response format results in an error.
+            ImageModel::GptImage1 => None,
+            ImageModel::GptImage1Mini => None,
+            ImageModel::GptImage1dot5 => None,
+            ImageModel::Other(_) => Some(ImageResponseFormat::B64Json),
+        };
+
+        let mut request_builder = CreateImageEditRequestArgs::default();
+
+        request_builder
+            .image(image_inputs)
+            .prompt(prompt.to_owned())
+            .model(model);
+
+        if let Some(size) = dalle2_size {
+            request_builder.size(size);
+        }
+
+        if let Some(response_format) = response_format {
+            request_builder.response_format(response_format);
+        }
+
+        let request = request_builder
+            .build()
+            .map_err(|e| anyhow::anyhow!("Failed to build CreateImageEditRequest: {}", e))?;
+
+        tracing::trace!(
+            model = format!("{:?}", request.model),
+            size = format!("{:?}", request.size),
+            response_format = format!("{:?}", request.response_format),
+            "Sending OpenAI image edit API request"
+        );
+
+        let response = self.client.images().edit(request).await?;
+
+        if let Some(image_data) = response.data.into_iter().next() {
+            match image_data.deref() {
+                Image::B64Json { b64_json, .. } => {
+                    let bytes = base64_decode(b64_json.as_ref())?;
+                    return Ok(ImageEditResult {
+                        bytes,
+                        mime_type: mxlink::mime::IMAGE_PNG,
+                    });
+                }
+                Image::Url { url, .. } => {
+                    tracing::warn!(?url, "Received URL instead of B64Json for image edit");
+                    return Err(anyhow::anyhow!(
+                        "Unexpected image type (URL) when B64Json was requested"
+                    ));
+                }
+            }
+        }
+
+        Err(anyhow::anyhow!(
+            "The OpenAI image edit API returned no images"
+        ))
+    }
+
     async fn text_to_speech(
         &self,
         input: &str,
@@ -333,7 +499,7 @@ impl ControllerTrait for Controller {
 
         let voice = if let Some(voice_string) = params.voice_override {
             // This is a hacky way to construct a Voice enum from the string we have.
-            let voice: serde_json::Result<async_openai::types::Voice> =
+            let voice: serde_json::Result<async_openai::types::audio::Voice> =
                 serde_json::from_str(&format!("\"{}\"", voice_string));
             match voice {
                 Ok(voice) => voice,
@@ -373,7 +539,7 @@ impl ControllerTrait for Controller {
             "Sending OpenAI text-to-speech API request"
         );
 
-        let result = self.client.audio().speech(request).await?;
+        let result = self.client.audio().speech().create(request).await?;
 
         Ok(TextToSpeechResult {
             bytes: result.bytes.into(),
@@ -391,20 +557,25 @@ impl ControllerTrait for Controller {
         }
     }
 
-    fn text_generation_prompt(&self) -> Option<String> {
-        let Some(text_generation_config) = &self.config.text_generation else {
-            return None;
-        };
+    fn text_generation_model_id(&self) -> Option<String> {
+        self.config
+            .text_generation
+            .as_ref()
+            .map(|config| config.model_id.to_owned())
+    }
 
-        text_generation_config.prompt.clone()
+    fn text_generation_prompt(&self) -> Option<String> {
+        self.config
+            .text_generation
+            .as_ref()
+            .and_then(|config| config.prompt.clone())
     }
 
     fn text_generation_temperature(&self) -> Option<f32> {
-        let Some(text_generation_config) = &self.config.text_generation else {
-            return None;
-        };
-
-        Some(text_generation_config.temperature)
+        self.config
+            .text_generation
+            .as_ref()
+            .map(|config| config.temperature)
     }
 
     fn text_to_speech_voice(&self) -> Option<String> {
@@ -427,15 +598,15 @@ impl ControllerTrait for Controller {
 }
 
 fn response_format_to_mime_type(
-    response_format: &async_openai::types::SpeechResponseFormat,
+    response_format: &async_openai::types::audio::SpeechResponseFormat,
 ) -> Option<mxlink::mime::Mime> {
     let content_type = match response_format {
-        async_openai::types::SpeechResponseFormat::Mp3 => "audio/mp3".to_owned(),
-        async_openai::types::SpeechResponseFormat::Wav => "audio/wav".to_owned(),
-        async_openai::types::SpeechResponseFormat::Opus => "audio/ogg".to_owned(),
-        async_openai::types::SpeechResponseFormat::Aac => "audio/aac".to_owned(),
-        async_openai::types::SpeechResponseFormat::Flac => "audio/flac".to_owned(),
-        async_openai::types::SpeechResponseFormat::Pcm => "audio/L8".to_owned(),
+        async_openai::types::audio::SpeechResponseFormat::Mp3 => "audio/mp3".to_owned(),
+        async_openai::types::audio::SpeechResponseFormat::Wav => "audio/wav".to_owned(),
+        async_openai::types::audio::SpeechResponseFormat::Opus => "audio/ogg".to_owned(),
+        async_openai::types::audio::SpeechResponseFormat::Aac => "audio/aac".to_owned(),
+        async_openai::types::audio::SpeechResponseFormat::Flac => "audio/flac".to_owned(),
+        async_openai::types::audio::SpeechResponseFormat::Pcm => "audio/L8".to_owned(),
     };
 
     match content_type.parse() {
@@ -463,3 +634,17 @@ fn audio_mime_type_to_file_name(mime_type: &mxlink::mime::Mime) -> Option<String
 
     Some(format!("audio.{}", file_extension))
 }
+
+/// Returns the smallest supported size for stickers based on what the image model supports.
+fn get_sticker_size(model: &ImageModel) -> async_openai::types::images::ImageSize {
+    use async_openai::types::images::ImageSize;
+
+    match model {
+        ImageModel::DallE2 => ImageSize::S256x256,
+        ImageModel::DallE3 => ImageSize::S1024x1024,
+        ImageModel::GptImage1 => ImageSize::S1024x1024,
+        ImageModel::GptImage1Mini => ImageSize::S1024x1024,
+        ImageModel::GptImage1dot5 => ImageSize::S1024x1024,
+        ImageModel::Other(_) => ImageSize::S1024x1024,
+    }
+}

src/agent/provider/openai/mod.rs

@@ -13,17 +13,19 @@ pub(super) use config::TextToSpeechConfig;
 
 use super::super::AgentInstantiationError;
 use super::super::AgentInstantiationResult;
-use super::controller::ControllerType;
 use super::ConfigTrait;
+use super::controller::ControllerType;
+
+pub const OPENAI_IMAGE_MODEL_GPT_IMAGE_1_DOT_5: &str = "gpt-image-1.5";
 
 pub fn create_controller_from_yaml_value_config(
     agent_id: &str,
-    config: serde_yaml::Value,
+    config: serde_yaml_ng::Value,
 ) -> AgentInstantiationResult<ControllerType> {
     let config = match &config {
-        serde_yaml::Value::Mapping(_) => {
+        serde_yaml_ng::Value::Mapping(_) => {
             let config: Config =
-                serde_yaml::from_value(config).map_err(AgentInstantiationError::Yaml)?;
+                serde_yaml_ng::from_value(config).map_err(AgentInstantiationError::Yaml)?;
 
             config
                 .validate()

src/agent/provider/openai/utils.rs

@@ -1,55 +1,48 @@
-use async_openai::types::{
-    ChatCompletionRequestAssistantMessageArgs, ChatCompletionRequestMessage,
-    ChatCompletionRequestSystemMessageArgs, ChatCompletionRequestUserMessageArgs,
+use async_openai::types::responses::{
+    EasyInputContent, EasyInputMessage, ImageDetail, InputContent, InputImageContent, InputItem,
+    InputParam, MessageType, Role,
 };
 
-use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage};
+use crate::conversation::llm::{
+    Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent,
+};
+use crate::utils::base64::base64_encode;
 
-pub fn convert_llm_messages_to_openai_messages(
+pub fn convert_llm_messages_to_openai_response_input(
     conversation_messages: Vec<LLMMessage>,
-) -> Vec<ChatCompletionRequestMessage> {
-    let mut openai_conversation_messages: Vec<ChatCompletionRequestMessage> =
-        Vec::with_capacity(conversation_messages.len());
+) -> InputParam {
+    let mut items = Vec::with_capacity(conversation_messages.len());
 
     for message in conversation_messages {
-        openai_conversation_messages.push(convert_llm_message_to_openai_message(message));
+        let role = match message.author {
+            LLMAuthor::Prompt => Role::System,
+            LLMAuthor::Assistant => Role::Assistant,
+            LLMAuthor::User => Role::User,
+        };
+
+        let content = match message.content {
+            LLMMessageContent::Text(text) => EasyInputContent::Text(text),
+            LLMMessageContent::Image(image_details) => {
+                let image_url = format!(
+                    "data:{};base64,{}",
+                    image_details.mime,
+                    base64_encode(&image_details.data)
+                );
+
+                EasyInputContent::ContentList(vec![InputContent::InputImage(InputImageContent {
+                    image_url: Some(image_url),
+                    detail: ImageDetail::Auto,
+                    file_id: None,
+                })])
+            }
+        };
+
+        items.push(InputItem::EasyMessage(EasyInputMessage {
+            r#type: MessageType::Message,
+            role,
+            content,
+        }));
     }
 
-    openai_conversation_messages
-}
-
-fn convert_llm_message_to_openai_message(llm_message: LLMMessage) -> ChatCompletionRequestMessage {
-    match llm_message.author {
-        LLMAuthor::Prompt => ChatCompletionRequestSystemMessageArgs::default()
-            .content(llm_message.message_text)
-            .build()
-            .expect("Failed building OpenAI system message")
-            .into(),
-        LLMAuthor::Assistant => ChatCompletionRequestAssistantMessageArgs::default()
-            .content(llm_message.message_text)
-            .build()
-            .expect("Failed building OpenAI assistant message")
-            .into(),
-        LLMAuthor::User => ChatCompletionRequestUserMessageArgs::default()
-            .content(llm_message.message_text)
-            .build()
-            .expect("Failed building OpenAI user message")
-            .into(),
-    }
-}
-
-pub(super) fn convert_string_to_enum<T>(value: &str) -> Result<T, String>
-where
-    T: serde::de::DeserializeOwned,
-{
-    // This is a hacky way to construct an enum from the string we have.
-    let enum_result: serde_json::Result<T> = serde_json::from_str(&format!("\"{}\"", value));
-    match enum_result {
-        Ok(enum_result) => Ok(enum_result),
-        Err(err) => {
-            tracing::debug!(?err, "Failed to parse into enum");
-
-            Err(format!("The value ({}) is not supported.", value))
-        }
-    }
+    InputParam::Items(items)
 }

src/agent/provider/openai_compat/config.rs

@@ -1,5 +1,6 @@
 use serde::{Deserialize, Serialize};
 
+use crate::agent::default_prompt;
 use crate::agent::provider::openai::{
     ImageGenerationConfig as OpenAIImageGenerationConfig,
     SpeechToTextConfig as OpenAISpeechToTextConfig,
@@ -65,7 +66,7 @@ pub struct TextGenerationConfig {
     pub temperature: f32,
 
     #[serde(default)]
-    pub max_response_tokens: u32,
+    pub max_response_tokens: Option<u32>,
 
     #[serde(default)]
     pub max_context_tokens: u32,
@@ -75,9 +76,9 @@ impl Default for TextGenerationConfig {
     fn default() -> Self {
         Self {
             model_id: default_text_model_id(),
-            prompt: Some("You are a brief, but helpful bot.".to_owned()),
+            prompt: Some(default_prompt().to_owned()),
             temperature: super::super::default_temperature(),
-            max_response_tokens: 4096,
+            max_response_tokens: Some(4096),
             max_context_tokens: 128_000,
         }
     }
@@ -92,7 +93,9 @@ impl TryInto<OpenAITextGenerationConfig> for TextGenerationConfig {
             prompt: self.prompt,
             temperature: self.temperature,
             max_response_tokens: self.max_response_tokens,
+            max_completion_tokens: None,
             max_context_tokens: self.max_context_tokens,
+            tools: Default::default(),
         })
     }
 }
@@ -159,13 +162,14 @@ impl TryInto<OpenAITextToSpeechConfig> for TextToSpeechConfig {
     type Error = String;
 
     fn try_into(self) -> Result<OpenAITextToSpeechConfig, Self::Error> {
-        let model_id = convert_string_to_enum::<async_openai::types::SpeechModel>(&self.model_id)?;
+        let model_id =
+            convert_string_to_enum::<async_openai::types::audio::SpeechModel>(&self.model_id)?;
 
-        let voice = convert_string_to_enum::<async_openai::types::Voice>(&self.voice)?;
+        let voice = convert_string_to_enum::<async_openai::types::audio::Voice>(&self.voice)?;
 
-        let response_format = convert_string_to_enum::<async_openai::types::SpeechResponseFormat>(
-            &self.response_format,
-        )?;
+        let response_format = convert_string_to_enum::<
+            async_openai::types::audio::SpeechResponseFormat,
+        >(&self.response_format)?;
 
         Ok(OpenAITextToSpeechConfig {
             model_id,
@@ -222,21 +226,27 @@ impl TryInto<OpenAIImageGenerationConfig> for ImageGenerationConfig {
 
     fn try_into(self) -> Result<OpenAIImageGenerationConfig, Self::Error> {
         let size = if let Some(size) = &self.size {
-            convert_string_to_enum::<async_openai::types::ImageSize>(size)?
+            Some(convert_string_to_enum::<
+                async_openai::types::images::ImageSize,
+            >(size)?)
         } else {
-            async_openai::types::ImageSize::S1024x1024
+            None
         };
 
         let style = if let Some(style) = &self.style {
-            convert_string_to_enum::<async_openai::types::ImageStyle>(style)?
+            Some(convert_string_to_enum::<
+                async_openai::types::images::ImageStyle,
+            >(style)?)
         } else {
-            async_openai::types::ImageStyle::Vivid
+            None
         };
 
         let quality = if let Some(quality) = &self.quality {
-            convert_string_to_enum::<async_openai::types::ImageQuality>(quality)?
+            Some(convert_string_to_enum::<
+                async_openai::types::images::ImageQuality,
+            >(quality)?)
         } else {
-            async_openai::types::ImageQuality::Standard
+            None
         };
 
         Ok(OpenAIImageGenerationConfig {

src/agent/provider/openai_compat/controller.rs

@@ -3,24 +3,28 @@ use etke_openai_api_rust::chat::{ChatApi, ChatBody};
 use etke_openai_api_rust::images::{ImagesApi, ImagesBody};
 use etke_openai_api_rust::{Auth, Message, OpenAI};
 
+const SMALLEST_IMAGE_SIZE: &str = "256x256";
+
 use super::super::ControllerTrait;
-use crate::agent::utils::base64_decode;
+use crate::utils::base64::base64_decode;
 use crate::{
     agent::provider::{
+        ImageEditParams, ImageGenerationParams, ImageSource, SpeechToTextParams,
+        SpeechToTextResult,
         entity::{TextGenerationParams, TextGenerationResult},
-        ImageGenerationParams, SpeechToTextParams, SpeechToTextResult,
     },
     conversation::llm::{
-        shorten_messages_list_to_context_size, Author as LLMAuthor,
-        Conversation as LLMConversation, Message as LLMMessage,
+        Author as LLMAuthor, Conversation as LLMConversation, Message as LLMMessage,
+        MessageContent as LLMMessageContent, shorten_messages_list_to_context_size,
     },
 };
 use crate::{
     agent::{
-        provider::entity::{
-            ImageGenerationResult, PingResult, TextToSpeechParams, TextToSpeechResult,
-        },
         AgentPurpose,
+        provider::entity::{
+            ImageEditResult, ImageGenerationResult, PingResult, TextToSpeechParams,
+            TextToSpeechResult,
+        },
     },
     strings,
 };
@@ -60,7 +64,8 @@ impl ControllerTrait for Controller {
 
         let messages = vec![LLMMessage {
             author: LLMAuthor::User,
-            message_text: "Hello!".to_string(),
+            content: LLMMessageContent::Text("Hello!".to_string()),
+            timestamp: chrono::Utc::now(),
         }];
 
         let conversation = LLMConversation { messages };
@@ -84,18 +89,20 @@ impl ControllerTrait for Controller {
             ));
         };
 
-        let prompt_text = params
-            .prompt_override
-            .unwrap_or(self.text_generation_prompt().unwrap_or("".to_owned()))
-            .trim()
-            .to_owned();
+        let prompt_text = params.prompt_variables.format(
+            params
+                .prompt_override
+                .unwrap_or(self.text_generation_prompt().unwrap_or("".to_owned()))
+                .trim(),
+        );
 
         let prompt_message = if prompt_text.is_empty() {
             None
         } else {
             Some(LLMMessage {
                 author: LLMAuthor::Prompt,
-                message_text: prompt_text,
+                content: LLMMessageContent::Text(prompt_text),
+                timestamp: chrono::Utc::now(),
             })
         };
 
@@ -130,12 +137,15 @@ impl ControllerTrait for Controller {
 
         let max_tokens = text_generation_config
             .max_response_tokens
-            .try_into()
-            .expect("Failed converting max_response_tokens from u32 to i32");
+            .map(|max_response_tokens| {
+                max_response_tokens
+                    .try_into()
+                    .expect("Failed converting max_response_tokens from u32 to i32")
+            });
 
         let request = ChatBody {
             model: text_generation_config.model_id.clone(),
-            max_tokens: Some(max_tokens),
+            max_tokens,
             temperature: Some(temperature),
             top_p: None,
             n: Some(1),
@@ -295,9 +305,11 @@ impl ControllerTrait for Controller {
         // when they span multiple lines.
         let prompt = prompt.replace("\n", " ");
 
-        let size: Option<String> = params
-            .size_override
-            .or_else(|| image_generation_config.size.clone());
+        let size: Option<String> = if params.smallest_size_possible {
+            Some(SMALLEST_IMAGE_SIZE.to_owned())
+        } else {
+            image_generation_config.size.clone()
+        };
 
         let request = ImagesBody {
             model: Some(image_generation_config.model_id.to_owned()),
@@ -360,6 +372,17 @@ impl ControllerTrait for Controller {
         ))
     }
 
+    async fn create_image_edit(
+        &self,
+        _prompt: &str,
+        _images: Vec<ImageSource>,
+        _params: ImageEditParams,
+    ) -> anyhow::Result<ImageEditResult> {
+        Err(anyhow::anyhow!(
+            "The OpenAI image edit API is not supported by the OpenAI-compat provider"
+        ))
+    }
+
     async fn text_to_speech(
         &self,
         input: &str,
@@ -409,20 +432,25 @@ impl ControllerTrait for Controller {
         }
     }
 
-    fn text_generation_prompt(&self) -> Option<String> {
-        let Some(text_generation_config) = &self.config.text_generation else {
-            return None;
-        };
+    fn text_generation_model_id(&self) -> Option<String> {
+        self.config
+            .text_generation
+            .as_ref()
+            .map(|config| config.model_id.to_owned())
+    }
 
-        text_generation_config.prompt.clone()
+    fn text_generation_prompt(&self) -> Option<String> {
+        self.config
+            .text_generation
+            .as_ref()
+            .and_then(|config| config.prompt.clone())
     }
 
     fn text_generation_temperature(&self) -> Option<f32> {
-        let Some(text_generation_config) = &self.config.text_generation else {
-            return None;
-        };
-
-        Some(text_generation_config.temperature)
+        self.config
+            .text_generation
+            .as_ref()
+            .map(|config| config.temperature)
     }
 
     fn text_to_speech_voice(&self) -> Option<String> {

src/agent/provider/openai_compat/mod.rs

@@ -21,17 +21,17 @@ pub use controller::Controller;
 
 use super::super::AgentInstantiationError;
 use super::super::AgentInstantiationResult;
-use super::controller::ControllerType;
 use super::ConfigTrait;
+use super::controller::ControllerType;
 
 pub fn create_controller_from_yaml_value_config(
     agent_id: &str,
-    config: serde_yaml::Value,
+    config: serde_yaml_ng::Value,
 ) -> AgentInstantiationResult<ControllerType> {
     let config = match &config {
-        serde_yaml::Value::Mapping(_) => {
+        serde_yaml_ng::Value::Mapping(_) => {
             let config: Config =
-                serde_yaml::from_value(config).map_err(AgentInstantiationError::Yaml)?;
+                serde_yaml_ng::from_value(config).map_err(AgentInstantiationError::Yaml)?;
 
             config
                 .validate()
@@ -56,7 +56,7 @@ pub fn default_config() -> Config {
 
     if let Some(text_generation) = &mut config.text_generation {
         text_generation.model_id = "some-model".to_string();
-        text_generation.max_response_tokens = 4096;
+        text_generation.max_response_tokens = Some(4096);
         text_generation.max_context_tokens = 128_000;
     }
 

src/agent/provider/openai_compat/utils.rs

@@ -2,7 +2,9 @@ use etke_openai_api_rust::{Message, Role};
 
 use crate::agent::provider::openai::Config as OpenAIConfig;
 
-use crate::conversation::llm::{Author as LLMAuthor, Message as LLMMessage};
+use crate::conversation::llm::{
+    Author as LLMAuthor, Message as LLMMessage, MessageContent as LLMMessageContent,
+};
 
 pub fn convert_llm_messages_to_openai_messages(
     conversation_messages: Vec<LLMMessage>,
@@ -11,22 +13,33 @@ pub fn convert_llm_messages_to_openai_messages(
         Vec::with_capacity(conversation_messages.len());
 
     for message in conversation_messages {
-        openai_conversation_messages.push(convert_llm_message_to_openai_message(message));
+        let openai_message = convert_llm_message_to_openai_message(message);
+        if let Some(openai_message) = openai_message {
+            openai_conversation_messages.push(openai_message);
+        }
     }
 
     openai_conversation_messages
 }
 
-fn convert_llm_message_to_openai_message(llm_message: LLMMessage) -> Message {
+fn convert_llm_message_to_openai_message(llm_message: LLMMessage) -> Option<Message> {
     let role = match llm_message.author {
         LLMAuthor::Prompt => Role::System,
         LLMAuthor::Assistant => Role::Assistant,
         LLMAuthor::User => Role::User,
     };
 
-    Message {
-        role,
-        content: llm_message.message_text,
+    match &llm_message.content {
+        LLMMessageContent::Text(text) => Some(Message {
+            role,
+            content: text.clone(),
+        }),
+        LLMMessageContent::Image(_image_details) => {
+            tracing::warn!(
+                "The OpenAI-compat provider's library does not support image content. This image message will be skipped."
+            );
+            None
+        }
     }
 }
 

src/agent/provider/openrouter/mod.rs

@@ -14,7 +14,7 @@ pub fn default_config() -> Config {
     if let Some(ref mut config) = config.text_generation.as_mut() {
         config.model_id = "mattshumer/reflection-70b:free".to_owned();
         config.max_context_tokens = 8192;
-        config.max_response_tokens = 2048;
+        config.max_response_tokens = Some(2048);
     }
 
     config

src/agent/provider/togetherai/mod.rs

@@ -14,7 +14,7 @@ pub fn default_config() -> Config {
     if let Some(ref mut config) = config.text_generation.as_mut() {
         config.model_id = "meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo".to_owned();
         config.max_context_tokens = 8192;
-        config.max_response_tokens = 2048;
+        config.max_response_tokens = Some(2048);
     }
 
     config

src/agent/utils.rs

@@ -1,5 +1,3 @@
-use base64::{engine::general_purpose::STANDARD, Engine as _};
-
 use crate::{
     agent::{
         AgentInstance, AgentPurpose, ControllerTrait, Manager as AgentManager, PublicIdentifier,
@@ -140,7 +138,3 @@ async fn get_global_agent_id_for_purpose(
         .handler
         .get_by_purpose_with_catch_all_fallback(purpose)
 }
-
-pub(crate) fn base64_decode(base64_string: &str) -> Result<Vec<u8>, base64::DecodeError> {
-    STANDARD.decode(base64_string)
-}

src/bot/implementation.rs

@@ -1,14 +1,17 @@
+use std::fs;
 use std::sync::Arc;
 use std::{future::Future, pin::Pin};
 
-use mxlink::matrix_sdk::media::{MediaFormat, MediaRequest};
-use mxlink::matrix_sdk::ruma::{
-    events::room::MediaSource, MilliSecondsSinceUnixEpoch, OwnedUserId,
-};
 use mxlink::matrix_sdk::Room;
+use mxlink::matrix_sdk::media::{MediaFormat, MediaRequestParameters};
+use mxlink::matrix_sdk::ruma::api::client::profile::{AvatarUrl, DisplayName};
+use mxlink::matrix_sdk::ruma::{
+    MilliSecondsSinceUnixEpoch, OwnedUserId, events::room::MediaSource,
+};
 
 use mxlink::{
     InitConfig, LoginConfig, LoginCredentials, LoginEncryption, MatrixLink, PersistenceConfig,
+    TypingNoticeGuard,
 };
 
 use mxlink::helpers::account_data_config::{
@@ -16,12 +19,13 @@ use mxlink::helpers::account_data_config::{
     RoomConfigManager as AccountDataRoomConfigManager,
 };
 use mxlink::helpers::encryption::Manager as EncryptionManager;
+use mxlink::mime::Mime;
 
 use crate::agent::Manager as AgentManager;
 use crate::entity::catch_up_marker::{
     CatchUpMarker, CatchUpMarkerManager, DelayedCatchUpMarkerManager,
 };
-use crate::entity::cfg::Config;
+use crate::entity::cfg::{Avatar, Config, ConfigUserAuth};
 use crate::entity::globalconfig::{GlobalConfig, GlobalConfigurationManager};
 use crate::entity::roomconfig::{RoomConfig, RoomConfigurationManager};
 
@@ -139,6 +143,10 @@ impl Bot {
         &self.inner.config.command_prefix
     }
 
+    pub(crate) fn post_join_self_introduction_enabled(&self) -> bool {
+        self.inner.config.room.post_join_self_introduction_enabled
+    }
+
     pub(crate) fn homeserver_name(&self) -> &str {
         &self.inner.config.homeserver.server_name
     }
@@ -171,6 +179,24 @@ impl Bot {
         self.matrix_link().user_id()
     }
 
+    pub(crate) async fn user_display_name_in_room(&self, room: &Room) -> Option<String> {
+        let bot_display_name = self
+            .room_display_name_fetcher()
+            .own_display_name_in_room(room)
+            .await;
+
+        match bot_display_name {
+            Ok(value) => value,
+            Err(err) => {
+                tracing::warn!(
+                    ?err,
+                    "Failed to fetch bot display name. Proceeding without it"
+                );
+                None
+            }
+        }
+    }
+
     pub(crate) fn reacting(&self) -> super::reacting::Reacting {
         super::reacting::Reacting::new(self.clone())
     }
@@ -210,6 +236,14 @@ impl Bot {
             .await
     }
 
+    pub(crate) async fn start_typing_notice(&self, room: &Room) -> TypingNoticeGuard {
+        self.inner
+            .matrix_link
+            .rooms()
+            .start_typing_notice(room)
+            .await
+    }
+
     pub async fn start(&self) -> anyhow::Result<()> {
         self.rooms().attach_event_handlers().await;
         self.messaging().attach_event_handlers().await;
@@ -254,24 +288,27 @@ impl Bot {
     async fn do_prepare_profile(&self) -> anyhow::Result<()> {
         tracing::debug!("Preparing profile..");
 
+        let desired_display_name = self.inner.config.user.name.clone();
+
         let account = self.inner.matrix_link.client().account();
         let media = self.inner.matrix_link.client().media();
 
-        let desired_display_name = self.inner.config.user.name.clone();
-
         let profile = account
-            .get_profile()
+            .fetch_user_profile()
             .await
             .map_err(|e| anyhow::anyhow!("Failed fetching profile: {:?}", e))?;
 
-        let should_update_display_name = match &profile.displayname {
+        let current_display_name = profile.get_static::<DisplayName>()?;
+        let current_avatar_url = profile.get_static::<AvatarUrl>()?;
+
+        let should_update_display_name = match &current_display_name {
             Some(displayname) => displayname != &desired_display_name,
             None => true,
         };
 
         if should_update_display_name {
             tracing::info!(
-                ?profile.displayname,
+                ?current_display_name,
                 ?desired_display_name,
                 "Updating display name.."
             );
@@ -281,34 +318,72 @@ impl Bot {
             }
         }
 
-        let should_update_avatar = match &profile.avatar_url {
-            Some(avatar_url) => {
-                let request = MediaRequest {
-                    source: MediaSource::Plain(avatar_url.to_owned()),
-                    format: MediaFormat::File,
-                };
-
-                let content = media
-                    .get_media_content(&request, true)
-                    .await
-                    .map_err(|e| anyhow::anyhow!("Failed fetching existing avatar: {:?}", e))?;
-
-                content.as_slice() != LOGO_BYTES
+        let desired_avatar: Option<(Vec<u8>, Mime)> = match &self.inner.config.user.avatar {
+            Avatar::Keep => {
+                tracing::info!("Avatar configured to keep current, skipping avatar management");
+                None
+            }
+            Avatar::Default => {
+                tracing::info!("Avatar configured to use default");
+                Some((
+                    LOGO_BYTES.to_vec(),
+                    LOGO_MIME_TYPE
+                        .parse()
+                        .expect("Failed parsing mime type for logo"),
+                ))
+            }
+            Avatar::Custom(avatar_path) => {
+                tracing::info!(?avatar_path, "Avatar configured to use custom path");
+                let bytes = fs::read(avatar_path).map_err(|e| {
+                    anyhow::anyhow!("Failed reading avatar from {:?}: {:?}", avatar_path, e)
+                })?;
+                let mime = mime_guess::from_path(avatar_path).first_or_octet_stream();
+                tracing::debug!(?mime, bytes_len = bytes.len(), "Loaded custom avatar");
+                Some((bytes, mime))
             }
-            None => true,
         };
 
-        if should_update_avatar {
-            tracing::info!("Updating avatar..");
+        if let Some((desired_bytes, mime_type)) = desired_avatar {
+            let should_update_avatar = match &current_avatar_url {
+                Some(avatar_url) => {
+                    tracing::debug!(?avatar_url, "Fetching current avatar to compare");
+                    let request = MediaRequestParameters {
+                        source: MediaSource::Plain(avatar_url.to_owned()),
+                        format: MediaFormat::File,
+                    };
 
-            let mime_type = LOGO_MIME_TYPE
-                .parse()
-                .expect("Failed parsing mime type for logo");
+                    let content = media
+                        .get_media_content(&request, true)
+                        .await
+                        .map_err(|e| anyhow::anyhow!("Failed fetching existing avatar: {:?}", e))?;
 
-            account
-                .upload_avatar(&mime_type, LOGO_BYTES.to_vec())
-                .await
-                .map_err(|e| anyhow::anyhow!("Failed uploading avatar: {:?}", e))?;
+                    let needs_update = content.as_slice() != desired_bytes;
+
+                    tracing::debug!(
+                        current_bytes_len = content.len(),
+                        desired_bytes_len = desired_bytes.len(),
+                        ?needs_update,
+                        "Compared current and desired avatar"
+                    );
+
+                    needs_update
+                }
+                None => {
+                    tracing::debug!("No current avatar set, will upload");
+                    true
+                }
+            };
+
+            if should_update_avatar {
+                tracing::info!("Updating avatar..");
+                account
+                    .upload_avatar(&mime_type, desired_bytes)
+                    .await
+                    .map_err(|e| anyhow::anyhow!("Failed uploading avatar: {:?}", e))?;
+                tracing::info!("Avatar updated successfully");
+            } else {
+                tracing::debug!("Avatar already up to date, skipping upload");
+            }
         }
 
         Ok(())
@@ -320,10 +395,22 @@ async fn create_matrix_link(config: &Config) -> anyhow::Result<MatrixLink> {
     let session_encryption_key = config.persistence.session_encryption_key()?;
     let db_dir_path: std::path::PathBuf = config.persistence.db_dir_path()?;
 
-    let login_creds = LoginCredentials::UserPassword(
-        config.user.mxid_localpart.to_owned(),
-        config.user.password.to_owned(),
-    );
+    let user_auth = config.user.auth_config(&config.homeserver.server_name)?;
+
+    let login_creds = match user_auth {
+        ConfigUserAuth::UserPassword { username, password } => {
+            LoginCredentials::UserPassword(username, password)
+        }
+        ConfigUserAuth::AccessToken {
+            user_id,
+            device_id,
+            access_token,
+        } => LoginCredentials::AccessToken {
+            user_id,
+            device_id,
+            access_token,
+        },
+    };
 
     let login_encryption = LoginEncryption::new(
         config.user.encryption.recovery_passphrase.clone(),

src/bot/load_config.rs

@@ -5,7 +5,7 @@ use anyhow::anyhow;
 
 use crate::agent::AgentPurpose;
 
-pub use crate::entity::cfg::{defaults as cfg_defaults, env as cfg_env, Config};
+pub use crate::entity::cfg::{Avatar, Config, defaults as cfg_defaults, env as cfg_env};
 
 pub fn load() -> anyhow::Result<Config> {
     let config_file_path = env::var(cfg_env::BAIBOT_CONFIG_FILE_PATH)
@@ -21,7 +21,7 @@ pub fn load() -> anyhow::Result<Config> {
     }
 
     let config_str = std::fs::read_to_string(config_file_path)?;
-    let mut config: Config = serde_yaml::from_str(&config_str)?;
+    let mut config: Config = serde_yaml_ng::from_str(&config_str)?;
 
     // Allow environment variables to override some configuration keys
     for (key, value) in env::vars() {
@@ -29,12 +29,29 @@ pub fn load() -> anyhow::Result<Config> {
             cfg_env::BAIBOT_HOMESERVER_SERVER_NAME => config.homeserver.server_name = value,
             cfg_env::BAIBOT_HOMESERVER_URL => config.homeserver.url = value,
             cfg_env::BAIBOT_USER_MXID_LOCALPART => config.user.mxid_localpart = value,
-            cfg_env::BAIBOT_USER_PASSWORD => config.user.password = value,
+            cfg_env::BAIBOT_USER_PASSWORD => {
+                config.user.password = optional_non_empty(value);
+            }
+            cfg_env::BAIBOT_USER_ACCESS_TOKEN => {
+                config.user.access_token = optional_non_empty(value);
+            }
+            cfg_env::BAIBOT_USER_DEVICE_ID => {
+                config.user.device_id = optional_non_empty(value);
+            }
             cfg_env::BAIBOT_USER_ENCRYPTION_RECOVERY_PASSPHRASE => {
                 config.user.encryption.recovery_passphrase = Some(value);
             }
+            cfg_env::BAIBOT_USER_ENCRYPTION_RECOVERY_RESET_ALLOWED => {
+                config.user.encryption.recovery_reset_allowed = value.parse::<bool>()?;
+            }
             cfg_env::BAIBOT_USER_NAME => config.user.name = value,
+            cfg_env::BAIBOT_USER_AVATAR => {
+                config.user.avatar = Avatar::from_string(value);
+            }
             cfg_env::BAIBOT_COMMAND_PREFIX => config.command_prefix = value,
+            cfg_env::BAIBOT_ROOM_POST_JOIN_SELF_INTRODUCTION_ENABLED => {
+                config.room.post_join_self_introduction_enabled = value.parse::<bool>()?;
+            }
             cfg_env::BAIBOT_LOGGING => {
                 config.logging = value;
             }
@@ -48,6 +65,9 @@ pub fn load() -> anyhow::Result<Config> {
             cfg_env::BAIBOT_PERSISTENCE_DATA_DIR_PATH => {
                 config.persistence.data_dir_path = Some(value);
             }
+            cfg_env::BAIBOT_PERSISTENCE_SESSION_ENCRYPTION_KEY => {
+                config.persistence.session_encryption_key = Some(value);
+            }
             cfg_env::BAIBOT_PERSISTENCE_CONFIG_ENCRYPTION_KEY => {
                 config.persistence.config_encryption_key = Some(value);
             }
@@ -108,3 +128,7 @@ pub fn load() -> anyhow::Result<Config> {
 
     Ok(config)
 }
+
+fn optional_non_empty(value: String) -> Option<String> {
+    if value.is_empty() { None } else { Some(value) }
+}

src/bot/messaging.rs

@@ -1,9 +1,9 @@
 use mxlink::matrix_sdk::{
-    ruma::{
-        api::client::receipt::create_receipt::v3::ReceiptType,
-        events::room::message::OriginalSyncRoomMessageEvent, OwnedEventId,
-    },
     Room,
+    ruma::{
+        OwnedEventId, api::client::receipt::create_receipt::v3::ReceiptType,
+        events::room::message::OriginalSyncRoomMessageEvent,
+    },
 };
 
 use mxlink::{CallbackError, MessageResponseType};
@@ -11,7 +11,7 @@ use mxlink::{CallbackError, MessageResponseType};
 use tracing::Instrument;
 
 use crate::{
-    conversation::matrix::determine_thread_context_for_room_event,
+    conversation::matrix::determine_interaction_context_for_room_event,
     entity::{MessageContext, MessagePayload, RoomConfigContext, TriggerEventInfo},
 };
 
@@ -239,8 +239,11 @@ impl Messaging {
             }
         };
 
-        let thread_context = determine_thread_context_for_room_event(
+        let bot_display_name = self.bot.user_display_name_in_room(&room).await;
+
+        let interaction_context = determine_interaction_context_for_room_event(
             self.bot.user_id(),
+            &bot_display_name,
             &room,
             &event,
             &payload,
@@ -248,16 +251,18 @@ impl Messaging {
         )
         .await;
 
-        let thread_context = match thread_context {
+        let interaction_context = match interaction_context {
             Ok(value) => value,
             Err(err) => {
-                tracing::error!(?err, "Failed to determine thread context for event");
+                tracing::error!(?err, "Failed to determine interaction context for event");
                 return Ok(());
             }
         };
 
-        let Some(thread_context) = thread_context else {
-            tracing::debug!("Ignoring message with unknown thread context (likely not a threaded message or a top-level message)");
+        let Some(interaction_context) = interaction_context else {
+            tracing::debug!(
+                "Ignoring message with unknown interaction context (likely not a message for us)"
+            );
             return Ok(());
         };
 
@@ -276,33 +281,14 @@ impl Messaging {
             room_config_context,
             self.bot.admin_pattern_regexes().clone(),
             trigger_event_info,
-            thread_context.info.clone(),
-        );
+            interaction_context.thread_info.clone(),
+        )
+        .with_bot_display_name(bot_display_name);
 
-        let bot_display_name = self
-            .bot
-            .room_display_name_fetcher()
-            .own_display_name_in_room(message_context.room())
-            .await;
-
-        let bot_display_name = match bot_display_name {
-            Ok(value) => value,
-            Err(err) => {
-                tracing::warn!(
-                    ?err,
-                    "Failed to fetch bot display name. Proceeding without it"
-                );
-                None
-            }
-        };
-
-        // The first event in the thread determines which handler processes the current event.
         let controller_type = crate::controller::determine_controller(
             self.bot.command_prefix(),
-            &thread_context.first_message,
+            &interaction_context.trigger,
             &message_context,
-            self.bot.user_id(),
-            &bot_display_name,
         );
 
         tracing::info!(?controller_type, "Determined controller");
@@ -310,7 +296,7 @@ impl Messaging {
         let _ = room
             .send_single_receipt(
                 ReceiptType::Read,
-                thread_context.info.clone().into(),
+                interaction_context.thread_info.clone().into(),
                 event.event_id.clone(),
             )
             .await;

src/bot/reacting.rs

@@ -1,12 +1,12 @@
 use mxlink::matrix_sdk::{
-    ruma::{
-        events::{
-            room::message::Relation, AnyMessageLikeEvent, AnySyncTimelineEvent, AnyTimelineEvent,
-            MessageLikeEvent,
-        },
-        OwnedEventId, OwnedUserId,
-    },
     Room,
+    ruma::{
+        OwnedEventId, OwnedUserId,
+        events::{
+            AnySyncMessageLikeEvent, AnySyncTimelineEvent, SyncMessageLikeEvent,
+            room::message::Relation,
+        },
+    },
 };
 
 use mxlink::CallbackError;
@@ -139,7 +139,7 @@ impl Reacting {
             }
         };
 
-        let reacted_to_event_any_timeline_event = match reacted_to_event.event.deserialize() {
+        let reacted_to_event_any_timeline_event = match reacted_to_event.raw().deserialize() {
             Ok(value) => value,
             Err(err) => {
                 tracing::error!(
@@ -154,7 +154,7 @@ impl Reacting {
         let reacted_to_event_sender_id: OwnedUserId =
             reacted_to_event_any_timeline_event.sender().to_owned();
 
-        let AnyTimelineEvent::MessageLike(reacted_to_event_message_like) =
+        let AnySyncTimelineEvent::MessageLike(reacted_to_event_message_like) =
             reacted_to_event_any_timeline_event
         else {
             tracing::debug!(
@@ -164,7 +164,7 @@ impl Reacting {
             return Ok(());
         };
 
-        let AnyMessageLikeEvent::RoomMessage(reacted_to_event_room_message) =
+        let AnySyncMessageLikeEvent::RoomMessage(reacted_to_event_room_message) =
             reacted_to_event_message_like
         else {
             tracing::debug!(
@@ -174,7 +174,7 @@ impl Reacting {
             return Ok(());
         };
 
-        let MessageLikeEvent::Original(reacted_to_event_room_message_original) =
+        let SyncMessageLikeEvent::Original(reacted_to_event_room_message_original) =
             reacted_to_event_room_message
         else {
             tracing::debug!(?reacted_to_event_id, "Ignoring redacted reacted-to event",);

src/bot/rooms.rs

@@ -1,9 +1,9 @@
 use mxlink::{
-    matrix_sdk::{
-        ruma::events::{room::member::StrippedRoomMemberEvent, AnySyncTimelineEvent},
-        Room,
-    },
     InvitationDecision,
+    matrix_sdk::{
+        Room,
+        ruma::events::{AnySyncTimelineEvent, room::member::StrippedRoomMemberEvent},
+    },
 };
 
 use mxlink::CallbackError;

src/controller/access/dispatching.rs

@@ -1,6 +1,6 @@
 use mxlink::MessageResponseType;
 
-use crate::{entity::MessageContext, strings, Bot};
+use crate::{Bot, entity::MessageContext, strings};
 
 use super::AccessControllerType;
 
@@ -13,7 +13,7 @@ pub async fn dispatch_controller(
     match handler {
         AccessControllerType::Help => {}
         _ => {
-            if !message_context.sender_can_manage_global_config()? {
+            if !message_context.sender_can_manage_global_config() {
                 bot.messaging()
                     .send_error_markdown_no_fail(
                         message_context.room(),

src/controller/access/help.rs

@@ -1,6 +1,6 @@
 use mxlink::MessageResponseType;
 
-use crate::{entity::MessageContext, strings, Bot};
+use crate::{Bot, entity::MessageContext, strings};
 
 pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> {
     let mut message = String::new();
@@ -80,24 +80,21 @@ fn build_section_users(
         message.push_str(&strings::access::users_no_patterns());
     }
 
-    let can_manage_global_config = message_context.sender_can_manage_global_config();
-    if let Ok(can_manage_global_config) = can_manage_global_config {
-        if can_manage_global_config {
-            message.push_str("\n\n");
+    if message_context.sender_can_manage_global_config() {
+        message.push_str("\n\n");
 
-            message.push_str(strings::the_following_commands_are_available());
-            message.push('\n');
+        message.push_str(strings::the_following_commands_are_available());
+        message.push('\n');
 
-            message.push_str(&strings::help::access::users_command_get(command_prefix));
-            message.push('\n');
+        message.push_str(&strings::help::access::users_command_get(command_prefix));
+        message.push('\n');
 
-            message.push_str(&strings::help::access::users_command_set(command_prefix));
-            message.push_str("\n\n");
+        message.push_str(&strings::help::access::users_command_set(command_prefix));
+        message.push_str("\n\n");
 
-            message.push_str(&strings::help::access::example_user_patterns(
-                homeserver_name,
-            ));
-        }
+        message.push_str(&strings::help::access::example_user_patterns(
+            homeserver_name,
+        ));
     }
 
     message
@@ -156,27 +153,24 @@ fn build_section_room_local_agent_managers(
         message.push_str(&strings::access::room_local_agent_managers_no_patterns());
     }
 
-    let can_manage_global_config = message_context.sender_can_manage_global_config();
-    if let Ok(can_manage_global_config) = can_manage_global_config {
-        if can_manage_global_config {
-            message.push_str("\n\n");
-            message.push_str(strings::the_following_commands_are_available());
-            message.push('\n');
+    if message_context.sender_can_manage_global_config() {
+        message.push_str("\n\n");
+        message.push_str(strings::the_following_commands_are_available());
+        message.push('\n');
 
-            message.push_str(
-                &strings::help::access::room_local_agent_managers_command_get(command_prefix),
-            );
-            message.push('\n');
+        message.push_str(
+            &strings::help::access::room_local_agent_managers_command_get(command_prefix),
+        );
+        message.push('\n');
 
-            message.push_str(
-                &strings::help::access::room_local_agent_managers_command_set(command_prefix),
-            );
-            message.push_str("\n\n");
+        message.push_str(
+            &strings::help::access::room_local_agent_managers_command_set(command_prefix),
+        );
+        message.push_str("\n\n");
 
-            message.push_str(&strings::help::access::example_user_patterns(
-                homeserver_name,
-            ));
-        }
+        message.push_str(&strings::help::access::example_user_patterns(
+            homeserver_name,
+        ));
     }
 
     message

src/controller/access/mod.rs

@@ -4,5 +4,5 @@ pub mod help;
 mod room_local_agent_managers;
 mod users;
 
-pub use determination::{determine_controller, AccessControllerType};
+pub use determination::{AccessControllerType, determine_controller};
 pub use dispatching::dispatch_controller;

src/controller/access/room_local_agent_managers.rs

@@ -1,6 +1,6 @@
 use mxlink::MessageResponseType;
 
-use crate::{entity::MessageContext, strings, Bot};
+use crate::{Bot, entity::MessageContext, strings};
 
 pub async fn handle_get(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> {
     let message = match &message_context
@@ -28,18 +28,18 @@ pub async fn handle_set(
     message_context: &MessageContext,
     patterns: &Option<Vec<String>>,
 ) -> anyhow::Result<()> {
-    if let Some(patterns) = patterns {
-        if let Err(err) = mxidwc::parse_patterns_vector(patterns) {
-            bot.messaging()
-                .send_error_markdown_no_fail(
-                    message_context.room(),
-                    &strings::access::failed_to_parse_patterns(&err.to_string()),
-                    MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()),
-                )
-                .await;
+    if let Some(patterns) = patterns
+        && let Err(err) = mxidwc::parse_patterns_vector(patterns)
+    {
+        bot.messaging()
+            .send_error_markdown_no_fail(
+                message_context.room(),
+                &strings::access::failed_to_parse_patterns(&err.to_string()),
+                MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()),
+            )
+            .await;
 
-            return Ok(());
-        }
+        return Ok(());
     }
 
     let mut global_config_manager_guard = bot.global_config_manager().lock().await;

src/controller/access/users.rs

@@ -1,6 +1,6 @@
 use mxlink::MessageResponseType;
 
-use crate::{entity::MessageContext, strings, Bot};
+use crate::{Bot, entity::MessageContext, strings};
 
 pub async fn handle_get(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> {
     let message = match &message_context.global_config().access.user_patterns {
@@ -24,18 +24,18 @@ pub async fn handle_set(
     message_context: &MessageContext,
     patterns: &Option<Vec<String>>,
 ) -> anyhow::Result<()> {
-    if let Some(patterns) = patterns {
-        if let Err(err) = mxidwc::parse_patterns_vector(patterns) {
-            bot.messaging()
-                .send_error_markdown_no_fail(
-                    message_context.room(),
-                    &strings::access::failed_to_parse_patterns(&err.to_string()),
-                    MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()),
-                )
-                .await;
+    if let Some(patterns) = patterns
+        && let Err(err) = mxidwc::parse_patterns_vector(patterns)
+    {
+        bot.messaging()
+            .send_error_markdown_no_fail(
+                message_context.room(),
+                &strings::access::failed_to_parse_patterns(&err.to_string()),
+                MessageResponseType::Reply(message_context.thread_info().root_event_id.clone()),
+            )
+            .await;
 
-            return Ok(());
-        }
+        return Ok(());
     }
 
     let mut global_config_manager_guard = bot.global_config_manager().lock().await;

src/controller/agent/create/mod.rs

@@ -3,19 +3,19 @@ mod tests;
 
 use mxlink::MessageResponseType;
 
-use crate::agent::provider::{ControllerTrait, PingResult};
 use crate::agent::PublicIdentifier;
-use crate::agent::{create_from_provider_and_yaml_value_config, AgentDefinition};
+use crate::agent::provider::{ControllerTrait, PingResult};
+use crate::agent::{AgentDefinition, create_from_provider_and_yaml_value_config};
 use crate::agent::{AgentInstance, AgentProvider};
 use crate::controller::utils::get_text_body_or_complain;
 use crate::entity::globalconfig::GlobalConfigurationManager;
 use crate::entity::roomconfig::RoomConfigurationManager;
 use crate::strings;
-use crate::{entity::MessageContext, Bot};
+use crate::{Bot, entity::MessageContext};
 
 struct ParsedAgentConfig {
     agent: AgentInstance,
-    config: serde_yaml::Value,
+    config: serde_yaml_ng::Value,
 }
 
 pub async fn handle_room_local(
@@ -100,8 +100,6 @@ pub async fn handle_room_local(
         return Ok(());
     };
 
-    message_context.room().typing_notice(true).await?;
-
     if !try_to_ping_agent_or_complain(bot, message_context, &parsed_config.agent).await {
         return Ok(());
     }
@@ -140,7 +138,7 @@ pub async fn handle_global(
     provider: &str,
     agent_id_prefixless: &str,
 ) -> anyhow::Result<()> {
-    if !message_context.sender_can_manage_global_config()? {
+    if !message_context.sender_can_manage_global_config() {
         bot.messaging()
             .send_error_markdown_no_fail(
                 message_context.room(),
@@ -215,8 +213,6 @@ pub async fn handle_global(
         return Ok(());
     };
 
-    message_context.room().typing_notice(true).await?;
-
     if !try_to_ping_agent_or_complain(bot, message_context, &parsed_config.agent).await {
         return Ok(());
     }
@@ -254,7 +250,7 @@ async fn send_guide(
     provider: &AgentProvider,
 ) -> anyhow::Result<()> {
     let sample_config = crate::agent::default_config_for_provider(provider);
-    let sample_config_pretty_yaml = serde_yaml::to_string(&sample_config)?;
+    let sample_config_pretty_yaml = serde_yaml_ng::to_string(&sample_config)?;
 
     bot.messaging()
         .send_text_markdown_no_fail(
@@ -267,7 +263,7 @@ async fn send_guide(
     Ok(())
 }
 
-fn parse_from_message_to_yaml_value(text: &str) -> Result<serde_yaml::Value, String> {
+fn parse_from_message_to_yaml_value(text: &str) -> Result<serde_yaml_ng::Value, String> {
     let mut text = text.trim();
 
     if text.starts_with("```") {
@@ -278,10 +274,10 @@ fn parse_from_message_to_yaml_value(text: &str) -> Result<serde_yaml::Value, Str
         text = text.trim_end_matches("```");
     }
 
-    let config: serde_yaml::Value = serde_yaml::from_str(text).map_err(|e| e.to_string())?;
+    let config: serde_yaml_ng::Value = serde_yaml_ng::from_str(text).map_err(|e| e.to_string())?;
 
     match config {
-        serde_yaml::Value::Mapping(_) => {}
+        serde_yaml_ng::Value::Mapping(_) => {}
         _ => {
             return Err("Not a valid YAML hashmap".to_owned());
         }

src/controller/agent/create/tests.rs

@@ -2,12 +2,12 @@
 fn agent_config_parsing_works() {
     struct TestCase {
         input: String,
-        expected: Option<serde_yaml::Value>,
+        expected: Option<serde_yaml_ng::Value>,
     }
 
     let provider = crate::agent::AgentProvider::OpenAI;
     let sample_config = crate::agent::default_config_for_provider(&provider);
-    let sample_config_pretty_yaml = serde_yaml::to_string(&sample_config).unwrap();
+    let sample_config_pretty_yaml = serde_yaml_ng::to_string(&sample_config).unwrap();
 
     let test_cases = vec![
         // Invalid input

src/controller/agent/delete/mod.rs

@@ -1,9 +1,9 @@
 use mxlink::MessageResponseType;
 
 use crate::entity::{
-    globalconfig::GlobalConfigurationManager, roomconfig::RoomConfigurationManager, MessageContext,
+    MessageContext, globalconfig::GlobalConfigurationManager, roomconfig::RoomConfigurationManager,
 };
-use crate::{agent::PublicIdentifier, strings, Bot};
+use crate::{Bot, agent::PublicIdentifier, strings};
 
 pub async fn handle(
     bot: &Bot,
@@ -50,7 +50,7 @@ pub async fn handle(
                 .await
         }
         PublicIdentifier::DynamicGlobal(_) => {
-            if !message_context.sender_can_manage_global_config()? {
+            if !message_context.sender_can_manage_global_config() {
                 bot.messaging()
                     .send_error_markdown_no_fail(
                         message_context.room(),

src/controller/agent/details/mod.rs

@@ -1,6 +1,6 @@
 use mxlink::MessageResponseType;
 
-use crate::{agent::PublicIdentifier, entity::MessageContext, strings, Bot};
+use crate::{Bot, agent::PublicIdentifier, entity::MessageContext, strings};
 
 pub async fn handle(
     bot: &Bot,
@@ -47,7 +47,7 @@ pub async fn handle(
             }
         }
         PublicIdentifier::DynamicGlobal(_) => {
-            if !message_context.sender_can_manage_global_config()? {
+            if !message_context.sender_can_manage_global_config() {
                 bot.messaging()
                     .send_error_markdown_no_fail(
                         message_context.room(),
@@ -64,7 +64,7 @@ pub async fn handle(
         PublicIdentifier::Static(_) => {}
     };
 
-    let config_yaml_pretty = serde_yaml::to_string(&agent.definition().config)?;
+    let config_yaml_pretty = serde_yaml_ng::to_string(&agent.definition().config)?;
 
     bot.messaging()
         .send_text_markdown_no_fail(

src/controller/agent/help/mod.rs

@@ -1,6 +1,6 @@
 use mxlink::MessageResponseType;
 
-use crate::{entity::MessageContext, strings, Bot};
+use crate::{Bot, entity::MessageContext, strings};
 
 pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> {
     // Anyone can access this help command, because certain subcommands ("list")
@@ -12,10 +12,7 @@ pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Resu
 
     message.push_str(&format!("## {}", strings::help::agent::heading()));
     message.push_str("\n\n");
-    message.push_str(&strings::help::agent::intro(
-        bot.command_prefix(),
-        can_manage_agents,
-    ));
+    message.push_str(&strings::help::agent::intro(bot.command_prefix()));
     message.push('\n');
     message.push_str(&strings::help::agent::intro_capabilities());
     message.push_str("\n\n");
@@ -39,7 +36,7 @@ pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Resu
         ));
         message.push('\n');
 
-        if message_context.sender_can_manage_global_config()? {
+        if message_context.sender_can_manage_global_config() {
             message.push_str(&strings::help::agent::create_agent_global(
                 bot.command_prefix(),
             ));

src/controller/agent/list/mod.rs

@@ -2,7 +2,7 @@ use mxlink::MessageResponseType;
 
 use crate::agent::AgentPurpose;
 use crate::strings;
-use crate::{entity::MessageContext, Bot};
+use crate::{Bot, entity::MessageContext};
 
 pub async fn handle(bot: &Bot, message_context: &MessageContext) -> anyhow::Result<()> {
     let agents = bot

src/controller/agent/mod.rs

@@ -1,4 +1,4 @@
-use crate::{entity::MessageContext, Bot};
+use crate::{Bot, entity::MessageContext};
 
 pub mod create;
 pub mod delete;
@@ -7,7 +7,7 @@ pub mod determination;
 pub mod help;
 pub mod list;
 
-pub use determination::{determine_controller, AgentControllerType};
+pub use determination::{AgentControllerType, determine_controller};
 
 pub async fn dispatch_controller(
     handler: &AgentControllerType,