Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
61d18b2e13 | ||
|
|
d831c08306 | ||
|
|
c70387b0c3 | ||
|
|
38516f2e17 | ||
|
|
5481b5a763 | ||
|
|
26bc437678 | ||
|
ec93f1ee2a |
@@ -1,3 +1,12 @@
|
||||
# (2026-02-04) Version 1.14.0
|
||||
|
||||
- (**Feature**) The `openai` provider now uses OpenAI's [Responses API](https://platform.openai.com/docs/api-reference/responses) (instead of the older Chat Completions API), adding support for [🛠️ built-in tools](./docs/features.md#️-built-in-tools-openai-only) (`web_search` and `code_interpreter`). These tools are **disabled by default** and can be enabled via the `text_generation.tools` configuration (see the [sample configuration](https://github.com/etkecc/baibot/blob/c70387b0c38d8d0f30bba2179a2a21a3710dbeaf/docs/sample-provider-configs/openai.yml#L12-L15)). To enable tools on an existing agent, you need to [update the agent](./docs/agents.md#updating-agents) to re-create it with the `text_generation.tools` section added and enable the tools you need. Thanks to [Layla Manley](https://github.com/yeslayla) for the contribution in [#62](https://github.com/etkecc/baibot/pull/62)!
|
||||
|
||||
- (**Bugfix**) Fix sticker generation for newer GPT image models (`gpt-image-1`, `gpt-image-1-mini`, `gpt-image-1.5`) which don't support the previously hardcoded `256x256` size (minimum is `1024x1024`)
|
||||
|
||||
- (**Internal Improvement**) Dependency updates
|
||||
|
||||
|
||||
# (2026-01-23) Version 1.13.0
|
||||
|
||||
- (**Improvement**) Extend auto-switching to support cheaper models (`gpt-image-1-mini`) for `gpt-image-1` and `gpt-image-1.5` when generating stickers ([e0b4a40](https://github.com/etkecc/baibot/commit/e0b4a40))
|
||||
|
||||
105
Cargo.lock
generated
105
Cargo.lock
generated
@@ -178,9 +178,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "async-openai"
|
||||
version = "0.32.3"
|
||||
version = "0.32.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "16aa4b9090c12270cd0925f735958ab6f0296af7ac08ce0f53fe320bcaf1e23d"
|
||||
checksum = "f4d91c2005832450ad9ac92411b9d4cec39e30be21adc76553efc47f812d59c7"
|
||||
dependencies = [
|
||||
"async-openai-macros",
|
||||
"backoff",
|
||||
@@ -288,7 +288,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "baibot"
|
||||
version = "1.13.0"
|
||||
version = "1.14.0"
|
||||
dependencies = [
|
||||
"anthropic",
|
||||
"anyhow",
|
||||
@@ -439,9 +439,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.11.0"
|
||||
version = "1.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3"
|
||||
checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
|
||||
|
||||
[[package]]
|
||||
name = "bytesize"
|
||||
@@ -460,9 +460,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.2.53"
|
||||
version = "1.2.55"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932"
|
||||
checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"shlex",
|
||||
@@ -1152,15 +1152,15 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
|
||||
|
||||
[[package]]
|
||||
name = "find-msvc-tools"
|
||||
version = "0.1.8"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.8"
|
||||
version = "1.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369"
|
||||
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
@@ -1679,14 +1679,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hyper-util"
|
||||
version = "0.1.19"
|
||||
version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f"
|
||||
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http 1.4.0",
|
||||
"http-body 1.0.1",
|
||||
@@ -1695,7 +1694,7 @@ dependencies = [
|
||||
"libc",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"socket2 0.6.1",
|
||||
"socket2 0.6.2",
|
||||
"tokio",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
@@ -1703,9 +1702,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.64"
|
||||
version = "0.1.65"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb"
|
||||
checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"core-foundation-sys",
|
||||
@@ -2578,9 +2577,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "num-conv"
|
||||
version = "0.1.0"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
|
||||
checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050"
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
@@ -2946,7 +2945,7 @@ dependencies = [
|
||||
"quinn-udp",
|
||||
"rustc-hash 2.1.1",
|
||||
"rustls 0.23.36",
|
||||
"socket2 0.6.1",
|
||||
"socket2 0.6.2",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tracing",
|
||||
@@ -2983,16 +2982,16 @@ dependencies = [
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"socket2 0.6.1",
|
||||
"socket2 0.6.2",
|
||||
"tracing",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.43"
|
||||
version = "1.0.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a"
|
||||
checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
@@ -3097,9 +3096,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.12.2"
|
||||
version = "1.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4"
|
||||
checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
@@ -3109,9 +3108,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.13"
|
||||
version = "0.4.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c"
|
||||
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
@@ -3120,9 +3119,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.8"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
|
||||
checksum = "a96887878f22d7bad8a3b6dc5b7440e0ada9a245242924394987b21cf2210a4c"
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
@@ -3880,15 +3879,15 @@ checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "1.0.1"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
|
||||
checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.11"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
|
||||
checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
@@ -3911,9 +3910,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "socket2"
|
||||
version = "0.6.1"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881"
|
||||
checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.60.2",
|
||||
@@ -4137,9 +4136,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.3.45"
|
||||
version = "0.3.46"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd"
|
||||
checksum = "9da98b7d9b7dad93488a84b8248efc35352b0b2657397d4167e7ad67e5d535e5"
|
||||
dependencies = [
|
||||
"deranged",
|
||||
"itoa",
|
||||
@@ -4152,15 +4151,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "time-core"
|
||||
version = "0.1.7"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca"
|
||||
checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca"
|
||||
|
||||
[[package]]
|
||||
name = "time-macros"
|
||||
version = "0.2.25"
|
||||
version = "0.2.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd"
|
||||
checksum = "78cc610bac2dcee56805c99642447d4c5dbde4d01f752ffea0199aee1f601dc4"
|
||||
dependencies = [
|
||||
"num-conv",
|
||||
"time-core",
|
||||
@@ -4201,7 +4200,7 @@ dependencies = [
|
||||
"libc",
|
||||
"mio",
|
||||
"pin-project-lite",
|
||||
"socket2 0.6.1",
|
||||
"socket2 0.6.2",
|
||||
"tokio-macros",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
@@ -4563,9 +4562,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.19.0"
|
||||
version = "1.20.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a"
|
||||
checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f"
|
||||
dependencies = [
|
||||
"getrandom 0.3.4",
|
||||
"js-sys",
|
||||
@@ -4774,14 +4773,14 @@ version = "0.26.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
|
||||
dependencies = [
|
||||
"webpki-roots 1.0.5",
|
||||
"webpki-roots 1.0.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c"
|
||||
checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
@@ -5147,18 +5146,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.33"
|
||||
version = "0.8.38"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd"
|
||||
checksum = "57cf3aa6855b23711ee9852dfc97dfaa51c45feaba5b645d0c777414d494a961"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.33"
|
||||
version = "0.8.38"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1"
|
||||
checksum = "8a616990af1a287837c4fe6596ad77ef57948f787e46ce28e166facc0cc1cb75"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5241,6 +5240,6 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.16"
|
||||
version = "1.0.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65"
|
||||
checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445"
|
||||
|
||||
@@ -7,7 +7,7 @@ license = "AGPL-3.0-or-later"
|
||||
readme = "README.md"
|
||||
keywords = ["matrix", "chat", "bot", "AI", "LLM"]
|
||||
include = ["/etc/assets/baibot-torso-768.png", "/src", "/README.md", "/CHANGELOG.md", "/LICENSE"]
|
||||
version = "1.13.0"
|
||||
version = "1.14.0"
|
||||
edition = "2024"
|
||||
|
||||
[lib]
|
||||
@@ -17,7 +17,7 @@ path = "src/lib.rs"
|
||||
[dependencies]
|
||||
anthropic = { git = "https://github.com/etkecc/anthropic-rs.git", branch = "fix-content-block-image" }
|
||||
anyhow = "1.0.*"
|
||||
async-openai = { version = "0.32.3", features = ["audio", "chat-completion", "image"] }
|
||||
async-openai = { version = "0.32.3", features = ["audio", "chat-completion", "image", "responses"] }
|
||||
base64 = "0.22.*"
|
||||
chrono = { version = "0.4.*", default-features = false, features = ["std", "now"] }
|
||||
# We'd rather not depend on this, but we cannot use the ruma-events EventContent macro without it.
|
||||
|
||||
@@ -17,7 +17,7 @@ It's influenced by [chaz](https://github.com/arcuru/chaz), but does **not** use
|
||||
|
||||
- Supports **different use purposes** (depending on the [☁️ provider](./docs/providers.md) & model):
|
||||
|
||||
- [💬 text-generation](./docs/features.md#-text-generation): communicating with you via text (though certain models may "see" images as well)
|
||||
- [💬 text-generation](./docs/features.md#-text-generation): communicating with you via text (though certain models may "see" images as well). The [OpenAI provider](./docs/providers.md#openai) also supports [🛠️ built-in tools](./docs/features.md#️-built-in-tools-openai-only) (web search, code interpreter)
|
||||
- [🦻 speech-to-text](./docs/features.md#-speech-to-text): turning your voice messages into text
|
||||
- [🗣️ text-to-speech](./docs/features.md#%EF%B8%8F-text-to-speech): turning bot or users text messages into voice messages
|
||||
- [🖌️ image-generation](./docs/features.md#image-generation): creating and editing images based on instructions
|
||||
|
||||
@@ -40,6 +40,23 @@ You may also wish to see:
|
||||
- [📖 Usage / 💬 Text Generation](./usage.md#-text-generation) section for more details on how to use the bot for Text Generation in a room
|
||||
|
||||
|
||||
#### 🛠️ Built-in Tools (OpenAI only)
|
||||
|
||||
|
||||
|
||||
The [OpenAI provider](./providers.md#openai) supports built-in tools that extend the model's capabilities:
|
||||
|
||||
- [🔍 Web Search](https://platform.openai.com/docs/guides/tools-web-search) (`web_search`): allows the model to search the web for up-to-date information. [🖼️ Screenshot](./screenshots/text-generation-tools-web-search.webp)
|
||||
|
||||
- [💻 Code Interpreter](https://platform.openai.com/docs/guides/tools-code-interpreter) (`code_interpreter`): allows the model to write and execute Python code in a sandbox
|
||||
|
||||
These tools are **disabled by default** and need to be explicitly enabled in the agent's `text_generation.tools` configuration. See the [OpenAI sample configuration](https://github.com/etkecc/baibot/blob/c70387b0c38d8d0f30bba2179a2a21a3710dbeaf/docs/sample-provider-configs/openai.yml#L12-L15) for reference.
|
||||
|
||||
To enable tools on an existing dynamically-created agent, you need to [update the agent](./agents.md#updating-agents) to re-create it with the `text_generation.tools` section added and enable the tools you need
|
||||
|
||||
💡 **Note**: These tools run on OpenAI's infrastructure and may incur additional costs. Web search results include citations that are incorporated into the response.
|
||||
|
||||
|
||||
#### On-demand involvement
|
||||
|
||||
In the following 2 cases, it's useful to involve the bot in conversations on-demand:
|
||||
|
||||
@@ -23,7 +23,7 @@ The list of supported providers is below.
|
||||
|
||||
### How to choose a provider
|
||||
|
||||
If you're not sure which provider to start with, **we recommend [OpenAI](#openai)** as it's the most popular and has the **widest range of capabilities**: [💬 text-generation](./features.md#-text-generation) (no vision), [🖌️ image-generation](./features.md#️image-generation), [🦻 speech-to-text](./features.md#-speech-to-text), [🗣️ text-to-speech](./features.md#️-text-to-speech).
|
||||
If you're not sure which provider to start with, **we recommend [OpenAI](#openai)** as it's the most popular and has the **widest range of capabilities**: [💬 text-generation](./features.md#-text-generation) (incl. vision, incl. [🛠️ tools](./features.md#️-built-in-tools-openai-only)), [🖌️ image-generation](./features.md#️image-generation), [🦻 speech-to-text](./features.md#-speech-to-text), [🗣️ text-to-speech](./features.md#️-text-to-speech).
|
||||
|
||||
You don't need to choose just one though. The bot supports [mixing & matching models](./features.md#-mixing--matching-models), so you can use multiple providers at the same time.
|
||||
|
||||
@@ -47,7 +47,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
|
||||
|
||||
- 🆔 Identifier: `anthropic`
|
||||
- 🔗 Links: [🏠 Home page](https://www.anthropic.com/), [🌐 Wiki](https://en.wikipedia.org/wiki/Anthropic), [👤 Sign up](https://console.anthropic.com/), [📋 Models list](https://docs.anthropic.com/en/docs/about-claude/models)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (incl. vision)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (incl. vision, no tools)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local anthropic my-anthropic-agent`
|
||||
- create a global agent: `!bai agent create-global anthropic my-anthropic-agent`
|
||||
@@ -61,7 +61,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
|
||||
|
||||
- 🆔 Identifier: `groq`
|
||||
- 🔗 Links: [🏠 Home page](https://groq.com/), [🌐 Wiki](https://en.wikipedia.org/wiki/Groq), [👤 Sign up](https://console.groq.com/login), [📋 Models list](https://console.groq.com/docs/models)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local groq my-groq-agent`
|
||||
- create a global agent: `!bai agent create-global groq my-groq-agent`
|
||||
@@ -75,7 +75,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
|
||||
|
||||
- 🆔 Identifier: `localai`
|
||||
- 🔗 Links: [🏠 Home page](https://localai.io/), [📋 Models list](https://localai.io/gallery.html)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local localai my-localai-agent`
|
||||
- create a global agent: `!bai agent create-global localai my-localai-agent`
|
||||
@@ -89,7 +89,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
|
||||
|
||||
- 🆔 Identifier: `mistral`
|
||||
- 🔗 Links: [🏠 Home page](https://mistral.ai/), [🌐 Wiki](https://en.wikipedia.org/wiki/Mistral_AI), [👤 Sign up](https://auth.mistral.ai/ui/registration), [📋 Models list](https://docs.mistral.ai/getting-started/models/)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local mistral my-mistral-agent`
|
||||
- create a global agent: `!bai agent create-global mistral my-mistral-agent`
|
||||
@@ -103,7 +103,7 @@ You don't need to choose just one though. The bot supports [mixing & matching mo
|
||||
|
||||
- 🆔 Identifier: `ollama`
|
||||
- 🔗 Links: [🏠 Home page](https://ollama.com/), [📋 Models list](https://ollama.com/library)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local ollama my-ollama-agent`
|
||||
- create a global agent: `!bai agent create-global ollama my-ollama-agent`
|
||||
@@ -120,7 +120,7 @@ For services which are not fully compatible with the OpenAI API, consider using
|
||||
|
||||
- 🆔 Identifier: `openai`
|
||||
- 🔗 Links: [🏠 Home page](https://openai.com/), [🌐 Wiki](https://en.wikipedia.org/wiki/OpenAI), [👤 Sign up](https://platform.openai.com/signup), [📋 Models list](https://platform.openai.com/docs/models)
|
||||
- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-creation), [💬 text-generation](./features.md#-text-generation) (incl. vision), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-creation), [💬 text-generation](./features.md#-text-generation) (incl. vision, incl. [🛠️ tools](./features.md#️-built-in-tools-openai-only)), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local openai my-openai-agent`
|
||||
- create a global agent: `!bai agent create-global openai my-openai-agent`
|
||||
@@ -137,7 +137,7 @@ Some of these popular services already have **shortcut** providers (leading to t
|
||||
This provider is just as featureful as the [OpenAI](#openai) provider, but is more compatible with services which do not fully adhere to the [OpenAI API spec](https://github.com/openai/openai-openapi/).
|
||||
|
||||
- 🆔 Identifier: `openai-compatible`
|
||||
- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-creation), [💬 text-generation](./features.md#-text-generation) (no vision), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🌟 Capabilities: [🖌️ image-generation](./features.md#️-image-creation), [💬 text-generation](./features.md#-text-generation) (no vision, no tools), [🗣️ text-to-speech](./features.md#️-text-to-speech), [🦻 speech-to-text](./features.md#-speech-to-text)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local openai-compatible my-openai-compatible-agent`
|
||||
- create a global agent: `!bai agent create-global openai-compatible my-openai-compatible-agent`
|
||||
@@ -151,7 +151,7 @@ This provider is just as featureful as the [OpenAI](#openai) provider, but is mo
|
||||
|
||||
- 🆔 Identifier: `openrouter`
|
||||
- 🔗 Links: [🏠 Home page](https://openrouter.ai/), [👤 Sign up](https://openrouter.ai/), [📋 Models list](https://openrouter.ai/models)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local openrouter my-openrouter-agent`
|
||||
- create a global agent: `!bai agent create-global openrouter my-openrouter-agent`
|
||||
@@ -165,7 +165,7 @@ This provider is just as featureful as the [OpenAI](#openai) provider, but is mo
|
||||
|
||||
- 🆔 Identifier: `together-ai`
|
||||
- 🔗 Links: [🏠 Home page](https://www.together.ai/), [👤 Sign up](https://api.together.ai/signup), [📋 Models list](https://api.together.xyz/models)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision)
|
||||
- 🌟 Capabilities: [💬 text-generation](./features.md#-text-generation) (no vision, no tools)
|
||||
- 🗲 Quick start:
|
||||
- create a room-local agent: `!bai agent create-room-local together-ai my-together-ai-agent`
|
||||
- create a global agent: `!bai agent create-global together-ai my-together-ai-agent`
|
||||
|
||||
@@ -9,6 +9,10 @@ text_generation:
|
||||
max_response_tokens: null
|
||||
max_completion_tokens: 128000
|
||||
max_context_tokens: 400000
|
||||
# Built-in tools
|
||||
tools:
|
||||
web_search: false
|
||||
code_interpreter: false
|
||||
speech_to_text:
|
||||
model_id: whisper-1
|
||||
text_to_speech:
|
||||
|
||||
BIN
docs/screenshots/text-generation-tools-web-search.webp
Normal file
BIN
docs/screenshots/text-generation-tools-web-search.webp
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 66 KiB |
@@ -90,6 +90,10 @@ agents:
|
||||
# max_response_tokens: null
|
||||
# max_completion_tokens: 128000
|
||||
# max_context_tokens: 400000
|
||||
# # Built-in tools
|
||||
# tools:
|
||||
# web_search: false
|
||||
# code_interpreter: false
|
||||
# speech_to_text:
|
||||
# model_id: whisper-1
|
||||
# text_to_speech:
|
||||
|
||||
@@ -14,7 +14,7 @@ services:
|
||||
- /etc/passwd:/etc/passwd:ro
|
||||
|
||||
synapse:
|
||||
image: ghcr.io/element-hq/synapse:v1.144.0
|
||||
image: ghcr.io/element-hq/synapse:v1.146.0
|
||||
user: "${UID}:${GID}"
|
||||
restart: unless-stopped
|
||||
entrypoint: python
|
||||
@@ -27,7 +27,7 @@ services:
|
||||
- ./synapse/media-store:/media-store
|
||||
|
||||
element-web:
|
||||
image: ghcr.io/element-hq/element-web:v1.12.7
|
||||
image: ghcr.io/element-hq/element-web:v1.12.9
|
||||
user: "${UID}:${GID}"
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
ollama:
|
||||
image: docker.io/ollama/ollama:0.13.5
|
||||
image: docker.io/ollama/ollama:0.15.4
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "${SERVICE_OLLAMA_BIND_PORT_HTTP}:11434"
|
||||
|
||||
@@ -69,6 +69,7 @@ impl AgentProvider {
|
||||
models_list_url: Some("https://docs.anthropic.com/en/docs/about-claude/models"),
|
||||
supported_purposes: vec![AgentPurpose::TextGeneration],
|
||||
text_generation_supports_vision: true,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
Self::Groq => AgentProviderInfo {
|
||||
id: Self::Groq.to_static_str(),
|
||||
@@ -80,11 +81,12 @@ impl AgentProvider {
|
||||
models_list_url: Some("https://console.groq.com/docs/models"),
|
||||
supported_purposes: vec![AgentPurpose::TextGeneration, AgentPurpose::SpeechToText],
|
||||
text_generation_supports_vision: false,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
Self::LocalAI => AgentProviderInfo {
|
||||
id: Self::LocalAI.to_static_str(),
|
||||
name: "LocalAI",
|
||||
description: "LocalAI is the free, Open Source OpenAI alternative. LocalAI act as a drop-in replacement REST API that’s compatible with OpenAI API specifications for local inferencing. It allows you to run LLMs, generate images, audio (and not only) locally or on-prem with consumer grade hardware, supporting multiple model families and architectures.",
|
||||
description: "LocalAI is the free, Open Source OpenAI alternative. LocalAI act as a drop-in replacement REST API that's compatible with OpenAI API specifications for local inferencing. It allows you to run LLMs, generate images, audio (and not only) locally or on-prem with consumer grade hardware, supporting multiple model families and architectures.",
|
||||
homepage_url: Some("https://localai.io/"),
|
||||
wiki_url: None,
|
||||
sign_up_url: None,
|
||||
@@ -95,6 +97,7 @@ impl AgentProvider {
|
||||
AgentPurpose::SpeechToText,
|
||||
],
|
||||
text_generation_supports_vision: false,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
Self::Mistral => AgentProviderInfo {
|
||||
id: Self::Mistral.to_static_str(),
|
||||
@@ -106,6 +109,7 @@ impl AgentProvider {
|
||||
models_list_url: Some("https://docs.mistral.ai/getting-started/models/"),
|
||||
supported_purposes: vec![AgentPurpose::TextGeneration],
|
||||
text_generation_supports_vision: false,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
Self::Ollama => AgentProviderInfo {
|
||||
id: Self::Ollama.to_static_str(),
|
||||
@@ -117,6 +121,7 @@ impl AgentProvider {
|
||||
models_list_url: Some("https://ollama.com/library"),
|
||||
supported_purposes: vec![AgentPurpose::TextGeneration],
|
||||
text_generation_supports_vision: false,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
Self::OpenAI => AgentProviderInfo {
|
||||
id: Self::OpenAI.to_static_str(),
|
||||
@@ -133,6 +138,7 @@ impl AgentProvider {
|
||||
AgentPurpose::SpeechToText,
|
||||
],
|
||||
text_generation_supports_vision: true,
|
||||
text_generation_supports_tools: true,
|
||||
},
|
||||
Self::OpenAICompat => AgentProviderInfo {
|
||||
id: Self::OpenAICompat.to_static_str(),
|
||||
@@ -149,6 +155,7 @@ impl AgentProvider {
|
||||
AgentPurpose::SpeechToText,
|
||||
],
|
||||
text_generation_supports_vision: false,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
Self::OpenRouter => AgentProviderInfo {
|
||||
id: Self::OpenRouter.to_static_str(),
|
||||
@@ -160,6 +167,7 @@ impl AgentProvider {
|
||||
models_list_url: Some("https://openrouter.ai/models"),
|
||||
supported_purposes: vec![AgentPurpose::TextGeneration],
|
||||
text_generation_supports_vision: false,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
Self::TogetherAI => AgentProviderInfo {
|
||||
id: Self::TogetherAI.to_static_str(),
|
||||
@@ -171,6 +179,7 @@ impl AgentProvider {
|
||||
models_list_url: Some("https://api.together.xyz/models"),
|
||||
supported_purposes: vec![AgentPurpose::TextGeneration],
|
||||
text_generation_supports_vision: false,
|
||||
text_generation_supports_tools: false,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -192,4 +201,5 @@ pub struct AgentProviderInfo {
|
||||
pub models_list_url: Option<&'static str>,
|
||||
pub supported_purposes: Vec<AgentPurpose>,
|
||||
pub text_generation_supports_vision: bool,
|
||||
pub text_generation_supports_tools: bool,
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ use mxlink::mime;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct ImageGenerationParams {
|
||||
pub size_override: Option<String>,
|
||||
pub smallest_size_possible: bool,
|
||||
|
||||
pub cheaper_model_switching_allowed: bool,
|
||||
|
||||
@@ -10,8 +10,8 @@ pub struct ImageGenerationParams {
|
||||
}
|
||||
|
||||
impl ImageGenerationParams {
|
||||
pub fn with_size_override(mut self, value: Option<String>) -> Self {
|
||||
self.size_override = value;
|
||||
pub fn with_smallest_size_possible(mut self, value: bool) -> Self {
|
||||
self.smallest_size_possible = value;
|
||||
self
|
||||
}
|
||||
|
||||
|
||||
@@ -64,6 +64,9 @@ pub struct TextGenerationConfig {
|
||||
|
||||
#[serde(default)]
|
||||
pub max_context_tokens: u32,
|
||||
|
||||
#[serde(default)]
|
||||
pub tools: ToolsConfig,
|
||||
}
|
||||
|
||||
impl Default for TextGenerationConfig {
|
||||
@@ -75,6 +78,7 @@ impl Default for TextGenerationConfig {
|
||||
max_response_tokens: None,
|
||||
max_completion_tokens: Some(128_000),
|
||||
max_context_tokens: 400_000,
|
||||
tools: ToolsConfig::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -83,6 +87,15 @@ fn default_text_model_id() -> String {
|
||||
"gpt-5.2".to_owned()
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ToolsConfig {
|
||||
#[serde(default)]
|
||||
pub web_search: bool,
|
||||
|
||||
#[serde(default)]
|
||||
pub code_interpreter: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SpeechToTextConfig {
|
||||
#[serde(default = "default_speech_to_text_model_id")]
|
||||
|
||||
@@ -5,7 +5,10 @@ use async_openai::{
|
||||
config::OpenAIConfig,
|
||||
types::{
|
||||
audio::{AudioInput, CreateSpeechRequestArgs, CreateTranscriptionRequestArgs},
|
||||
chat::{ChatCompletionRequestMessage, CreateChatCompletionRequestArgs},
|
||||
responses::{
|
||||
CodeInterpreterContainerAuto, CodeInterpreterTool, CodeInterpreterToolContainer,
|
||||
CreateResponseArgs, OutputItem, OutputMessageContent, Tool, WebSearchTool,
|
||||
},
|
||||
images::{
|
||||
CreateImageEditRequestArgs, CreateImageRequestArgs,
|
||||
Image, ImageInput, ImageModel, ImageResponseFormat,
|
||||
@@ -28,13 +31,10 @@ use crate::{
|
||||
use crate::{
|
||||
agent::{
|
||||
AgentPurpose,
|
||||
provider::{
|
||||
entity::{
|
||||
provider::entity::{
|
||||
ImageEditResult, ImageGenerationResult, ImageSource, PingResult,
|
||||
TextToSpeechParams, TextToSpeechResult,
|
||||
},
|
||||
openai::utils::convert_string_to_enum,
|
||||
},
|
||||
},
|
||||
strings,
|
||||
};
|
||||
@@ -129,28 +129,44 @@ impl ControllerTrait for Controller {
|
||||
conversation_messages.insert(0, prompt_message);
|
||||
}
|
||||
|
||||
let openai_conversation_messages: Vec<ChatCompletionRequestMessage> =
|
||||
super::utils::convert_llm_messages_to_openai_messages(conversation_messages);
|
||||
let input = super::utils::convert_llm_messages_to_openai_response_input(conversation_messages);
|
||||
|
||||
let messages_count = openai_conversation_messages.len();
|
||||
let messages_count = match &input {
|
||||
async_openai::types::responses::InputParam::Items(items) => items.len(),
|
||||
_ => 1,
|
||||
};
|
||||
|
||||
let temperature = params
|
||||
.temperature_override
|
||||
.unwrap_or(text_generation_config.temperature);
|
||||
|
||||
let mut request_builder = CreateChatCompletionRequestArgs::default();
|
||||
let mut request_builder = CreateResponseArgs::default();
|
||||
|
||||
request_builder
|
||||
.model(&text_generation_config.model_id)
|
||||
.temperature(temperature)
|
||||
.messages(openai_conversation_messages);
|
||||
.input(input);
|
||||
|
||||
if let Some(max_response_tokens) = text_generation_config.max_response_tokens {
|
||||
request_builder.max_tokens(max_response_tokens);
|
||||
let mut tools = Vec::new();
|
||||
if text_generation_config.tools.web_search {
|
||||
tools.push(Tool::WebSearch(WebSearchTool::default()));
|
||||
}
|
||||
if text_generation_config.tools.code_interpreter {
|
||||
tools.push(Tool::CodeInterpreter(CodeInterpreterTool {
|
||||
container: CodeInterpreterToolContainer::Auto(
|
||||
CodeInterpreterContainerAuto::default(),
|
||||
),
|
||||
}));
|
||||
}
|
||||
|
||||
if let Some(max_completion_tokens) = text_generation_config.max_completion_tokens {
|
||||
request_builder.max_completion_tokens(max_completion_tokens);
|
||||
if !tools.is_empty() {
|
||||
request_builder.tools(tools);
|
||||
}
|
||||
|
||||
if let Some(max_response_tokens) = text_generation_config.max_response_tokens {
|
||||
request_builder.max_output_tokens(max_response_tokens);
|
||||
} else if let Some(max_completion_tokens) = text_generation_config.max_completion_tokens {
|
||||
request_builder.max_output_tokens(max_completion_tokens);
|
||||
}
|
||||
|
||||
let request = request_builder.build()?;
|
||||
@@ -160,33 +176,31 @@ impl ControllerTrait for Controller {
|
||||
model = format!("{:?}", request.model),
|
||||
?messages_count,
|
||||
request = request_as_json,
|
||||
"Sending OpenAI chat completion API request"
|
||||
"Sending OpenAI response API request"
|
||||
);
|
||||
}
|
||||
|
||||
let response = self.client.chat().create(request).await?;
|
||||
let response = self.client.responses().create(request).await?;
|
||||
|
||||
tracing::trace!(
|
||||
?response,
|
||||
"Got response from the OpenAI chat completion API"
|
||||
"Got response from the OpenAI response API"
|
||||
);
|
||||
|
||||
// We only request 1 result, so there should only be 1 choice.
|
||||
if let Some(choice) = response.choices.into_iter().next() {
|
||||
match choice.message.content {
|
||||
Some(text) => {
|
||||
return Ok(TextGenerationResult { text });
|
||||
for item in response.output {
|
||||
if let OutputItem::Message(message) = item {
|
||||
for content in message.content {
|
||||
if let OutputMessageContent::OutputText(text_content) = content {
|
||||
return Ok(TextGenerationResult {
|
||||
text: text_content.text,
|
||||
});
|
||||
}
|
||||
None => {
|
||||
return Err(anyhow::anyhow!(
|
||||
"No content was found in the response choice from the OpenAI chat completion API"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(anyhow::anyhow!(
|
||||
"No response messages choices were returned from the OpenAI chat completion API"
|
||||
"No response messages choices were returned from the OpenAI response API"
|
||||
))
|
||||
}
|
||||
|
||||
@@ -295,10 +309,11 @@ impl ControllerTrait for Controller {
|
||||
image_generation_config.quality.clone()
|
||||
};
|
||||
|
||||
let size = params
|
||||
.size_override
|
||||
.map(|s| convert_string_to_enum::<async_openai::types::images::ImageSize>(&s).unwrap())
|
||||
.or(image_generation_config.size);
|
||||
let size = if params.smallest_size_possible {
|
||||
Some(get_sticker_size(&model))
|
||||
} else {
|
||||
image_generation_config.size
|
||||
};
|
||||
|
||||
let response_format = match model.clone() {
|
||||
ImageModel::DallE2 => Some(ImageResponseFormat::B64Json),
|
||||
@@ -621,3 +636,17 @@ fn audio_mime_type_to_file_name(mime_type: &mxlink::mime::Mime) -> Option<String
|
||||
|
||||
Some(format!("audio.{}", file_extension))
|
||||
}
|
||||
|
||||
/// Returns the smallest supported size for stickers based on what the image model supports.
|
||||
fn get_sticker_size(model: &ImageModel) -> async_openai::types::images::ImageSize {
|
||||
use async_openai::types::images::ImageSize;
|
||||
|
||||
match model {
|
||||
ImageModel::DallE2 => ImageSize::S256x256,
|
||||
ImageModel::DallE3 => ImageSize::S1024x1024,
|
||||
ImageModel::GptImage1 => ImageSize::S1024x1024,
|
||||
ImageModel::GptImage1Mini => ImageSize::S1024x1024,
|
||||
ImageModel::GptImage1dot5 => ImageSize::S1024x1024,
|
||||
ImageModel::Other(_) => ImageSize::S1024x1024,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,6 @@
|
||||
use async_openai::types::{
|
||||
chat::{
|
||||
ChatCompletionRequestAssistantMessageArgs, ChatCompletionRequestMessage,
|
||||
ChatCompletionRequestMessageContentPartImage,
|
||||
ChatCompletionRequestSystemMessageArgs,
|
||||
ChatCompletionRequestUserMessageArgs, ChatCompletionRequestUserMessageContent,
|
||||
ChatCompletionRequestUserMessageContentPart,
|
||||
ImageUrlArgs,
|
||||
},
|
||||
use async_openai::types::responses::{
|
||||
EasyInputContent, EasyInputMessage, ImageDetail, InputContent, InputImageContent, InputItem,
|
||||
InputParam, MessageType, Role,
|
||||
};
|
||||
|
||||
use crate::conversation::llm::{
|
||||
@@ -14,43 +8,20 @@ use crate::conversation::llm::{
|
||||
};
|
||||
use crate::utils::base64::base64_encode;
|
||||
|
||||
pub fn convert_llm_messages_to_openai_messages(
|
||||
pub fn convert_llm_messages_to_openai_response_input(
|
||||
conversation_messages: Vec<LLMMessage>,
|
||||
) -> Vec<ChatCompletionRequestMessage> {
|
||||
let mut openai_conversation_messages: Vec<ChatCompletionRequestMessage> =
|
||||
Vec::with_capacity(conversation_messages.len());
|
||||
) -> InputParam {
|
||||
let mut items = Vec::with_capacity(conversation_messages.len());
|
||||
|
||||
for message in conversation_messages {
|
||||
let openai_message = convert_llm_message_to_openai_message(message);
|
||||
if let Some(openai_message) = openai_message {
|
||||
openai_conversation_messages.push(openai_message);
|
||||
}
|
||||
}
|
||||
let role = match message.author {
|
||||
LLMAuthor::Prompt => Role::System,
|
||||
LLMAuthor::Assistant => Role::Assistant,
|
||||
LLMAuthor::User => Role::User,
|
||||
};
|
||||
|
||||
openai_conversation_messages
|
||||
}
|
||||
|
||||
fn convert_llm_message_to_openai_message(
|
||||
llm_message: LLMMessage,
|
||||
) -> Option<ChatCompletionRequestMessage> {
|
||||
match &llm_message.content {
|
||||
LLMMessageContent::Text(text) => Some(match llm_message.author {
|
||||
LLMAuthor::Prompt => ChatCompletionRequestSystemMessageArgs::default()
|
||||
.content(text.clone())
|
||||
.build()
|
||||
.expect("Failed building OpenAI system message")
|
||||
.into(),
|
||||
LLMAuthor::Assistant => ChatCompletionRequestAssistantMessageArgs::default()
|
||||
.content(text.clone())
|
||||
.build()
|
||||
.expect("Failed building OpenAI assistant message")
|
||||
.into(),
|
||||
LLMAuthor::User => ChatCompletionRequestUserMessageArgs::default()
|
||||
.content(text.clone())
|
||||
.build()
|
||||
.expect("Failed building OpenAI user message")
|
||||
.into(),
|
||||
}),
|
||||
let content = match message.content {
|
||||
LLMMessageContent::Text(text) => EasyInputContent::Text(text),
|
||||
LLMMessageContent::Image(image_details) => {
|
||||
let image_url = format!(
|
||||
"data:{};base64,{}",
|
||||
@@ -58,49 +29,20 @@ fn convert_llm_message_to_openai_message(
|
||||
base64_encode(&image_details.data)
|
||||
);
|
||||
|
||||
let part = ChatCompletionRequestUserMessageContentPart::ImageUrl(
|
||||
ChatCompletionRequestMessageContentPartImage {
|
||||
image_url: ImageUrlArgs::default()
|
||||
.url(image_url)
|
||||
.build()
|
||||
.expect("Failed building OpenAI image url"),
|
||||
},
|
||||
);
|
||||
EasyInputContent::ContentList(vec![InputContent::InputImage(InputImageContent {
|
||||
image_url: Some(image_url),
|
||||
detail: ImageDetail::Auto,
|
||||
file_id: None,
|
||||
})])
|
||||
}
|
||||
};
|
||||
|
||||
let message_content = ChatCompletionRequestUserMessageContent::Array(vec![part]);
|
||||
items.push(InputItem::EasyMessage(EasyInputMessage {
|
||||
r#type: MessageType::Message,
|
||||
role,
|
||||
content,
|
||||
}));
|
||||
}
|
||||
|
||||
match llm_message.author {
|
||||
LLMAuthor::User => Some(
|
||||
ChatCompletionRequestUserMessageArgs::default()
|
||||
.content(message_content)
|
||||
.build()
|
||||
.expect("Failed building OpenAI user message")
|
||||
.into(),
|
||||
),
|
||||
_ => {
|
||||
tracing::warn!(
|
||||
"OpenAI API does not support image content for messages authored by {:?}. This message part will be skipped.",
|
||||
llm_message.author
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn convert_string_to_enum<T>(value: &str) -> Result<T, String>
|
||||
where
|
||||
T: serde::de::DeserializeOwned,
|
||||
{
|
||||
// This is a hacky way to construct an enum from the string we have.
|
||||
let enum_result: serde_json::Result<T> = serde_json::from_str(&format!("\"{}\"", value));
|
||||
match enum_result {
|
||||
Ok(enum_result) => Ok(enum_result),
|
||||
Err(err) => {
|
||||
tracing::debug!(?err, "Failed to parse into enum");
|
||||
|
||||
Err(format!("The value ({}) is not supported.", value))
|
||||
}
|
||||
}
|
||||
InputParam::Items(items)
|
||||
}
|
||||
|
||||
@@ -95,6 +95,7 @@ impl TryInto<OpenAITextGenerationConfig> for TextGenerationConfig {
|
||||
max_response_tokens: self.max_response_tokens,
|
||||
max_completion_tokens: None,
|
||||
max_context_tokens: self.max_context_tokens,
|
||||
tools: Default::default(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,8 @@ use etke_openai_api_rust::chat::{ChatApi, ChatBody};
|
||||
use etke_openai_api_rust::images::{ImagesApi, ImagesBody};
|
||||
use etke_openai_api_rust::{Auth, Message, OpenAI};
|
||||
|
||||
const SMALLEST_IMAGE_SIZE: &str = "256x256";
|
||||
|
||||
use super::super::ControllerTrait;
|
||||
use crate::utils::base64::base64_decode;
|
||||
use crate::{
|
||||
@@ -303,9 +305,11 @@ impl ControllerTrait for Controller {
|
||||
// when they span multiple lines.
|
||||
let prompt = prompt.replace("\n", " ");
|
||||
|
||||
let size: Option<String> = params
|
||||
.size_override
|
||||
.or_else(|| image_generation_config.size.clone());
|
||||
let size: Option<String> = if params.smallest_size_possible {
|
||||
Some(SMALLEST_IMAGE_SIZE.to_owned())
|
||||
} else {
|
||||
image_generation_config.size.clone()
|
||||
};
|
||||
|
||||
let request = ImagesBody {
|
||||
model: Some(image_generation_config.model_id.to_owned()),
|
||||
|
||||
@@ -12,9 +12,6 @@ use crate::strings;
|
||||
use crate::utils::mime::get_file_extension;
|
||||
use crate::{Bot, entity::MessageContext};
|
||||
|
||||
// We may make this configurable (per room, etc.) in the future, but for now it's hardcoded.
|
||||
const STICKER_SIZE: &str = "256x256";
|
||||
|
||||
pub async fn handle_image(
|
||||
bot: &Bot,
|
||||
matrix_link: MatrixLink,
|
||||
@@ -177,7 +174,7 @@ pub async fn handle_sticker(
|
||||
);
|
||||
|
||||
let params = ImageGenerationParams::default()
|
||||
.with_size_override(Some(STICKER_SIZE.to_owned()))
|
||||
.with_smallest_size_possible(true)
|
||||
.with_cheaper_model_switching_allowed(true)
|
||||
.with_cheaper_quality_switching_allowed(true);
|
||||
|
||||
|
||||
@@ -109,11 +109,21 @@ pub fn help_provider_details(id: &str, info: &AgentProviderInfo) -> String {
|
||||
let mut purpose_line = format!("{} {}", purpose.emoji(), purpose.as_str());
|
||||
|
||||
if let AgentPurpose::TextGeneration = purpose {
|
||||
let mut extras = vec![];
|
||||
|
||||
if info.text_generation_supports_vision {
|
||||
purpose_line = format!("{} ({})", purpose_line, "incl. vision");
|
||||
extras.push("incl. vision");
|
||||
} else {
|
||||
purpose_line = format!("{} ({})", purpose_line, "no vision");
|
||||
extras.push("no vision");
|
||||
}
|
||||
|
||||
if info.text_generation_supports_tools {
|
||||
extras.push("incl. tools");
|
||||
} else {
|
||||
extras.push("no tools");
|
||||
}
|
||||
|
||||
purpose_line = format!("{} ({})", purpose_line, extras.join(", "));
|
||||
}
|
||||
|
||||
capabilities.push(purpose_line);
|
||||
|
||||
@@ -64,7 +64,7 @@ To create a sticker, send a command like `%command_prefix% sticker A huge bowl o
|
||||
|
||||
The difference from **creating images** is that the bot will:
|
||||
|
||||
- create a smaller-resolution image (`256x256`) - smaller/quicker, but still good enough for a sticker
|
||||
- create a smaller-resolution image (as small as the model allows) - smaller/quicker, but still good enough for a sticker
|
||||
- potentially switch to a different (cheaper or otherwise more suitable) model, if available
|
||||
- post the image directly to the room (as a reply to your message), without starting a threaded conversation
|
||||
"#;
|
||||
|
||||
Reference in New Issue
Block a user