unfunny/baibot-withmcp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update tiktoken-rs to 0.11, adding support for newer GPT models

Browse Source 
Supersedes https://github.com/etkecc/baibot/pull/116

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Slavi Pantaleev
2026-04-11 10:52:22 +03:00
parent 1563e83672
commit a865a26093
3 changed files with 33 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
Cargo.lock generated
View File

@@ -339,18 +339,18 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
[[package]] [[package]]
name = "bit-set" name = "bit-set"
version = "0.5.3" version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
dependencies = [ dependencies = [
"bit-vec", "bit-vec",
] ]
[[package]] [[package]]
name = "bit-vec" name = "bit-vec"
version = "0.6.3" version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
@@ -1048,7 +1048,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [ dependencies = [
"libc", "libc",
"windows-sys 0.52.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -1137,9 +1137,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]] [[package]]
name = "fancy-regex" name = "fancy-regex"
version = "0.13.0" version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"
dependencies = [ dependencies = [
"bit-set", "bit-set",
"regex-automata", "regex-automata",
@@ -1738,7 +1738,7 @@ dependencies = [
"libc", "libc",
"percent-encoding", "percent-encoding",
"pin-project-lite", "pin-project-lite",
"socket2 0.5.10", "socket2 0.6.3",
"tokio", "tokio",
"tower-service", "tower-service",
"tracing", "tracing",
@@ -2631,7 +2631,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [ dependencies = [
"windows-sys 0.60.2", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -3008,7 +3008,7 @@ dependencies = [
"quinn-udp", "quinn-udp",
"rustc-hash 2.1.2", "rustc-hash 2.1.2",
"rustls 0.23.37", "rustls 0.23.37",
"socket2 0.5.10", "socket2 0.6.3",
"thiserror 2.0.18", "thiserror 2.0.18",
"tokio", "tokio",
"tracing", "tracing",
@@ -3045,7 +3045,7 @@ dependencies = [
"cfg_aliases", "cfg_aliases",
"libc", "libc",
"once_cell", "once_cell",
"socket2 0.5.10", "socket2 0.6.3",
"tracing", "tracing",
"windows-sys 0.60.2", "windows-sys 0.60.2",
] ]
@@ -3595,7 +3595,7 @@ dependencies = [
"errno", "errno",
"libc", "libc",
"linux-raw-sys", "linux-raw-sys",
"windows-sys 0.52.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -4001,7 +4001,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [ dependencies = [
"libc", "libc",
"windows-sys 0.60.2", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -4142,7 +4142,7 @@ dependencies = [
"getrandom 0.4.2", "getrandom 0.4.2",
"once_cell", "once_cell",
"rustix", "rustix",
"windows-sys 0.52.0", "windows-sys 0.61.2",
] ]
[[package]] [[package]]
@@ -4207,9 +4207,9 @@ dependencies = [
[[package]] [[package]]
name = "tiktoken-rs" name = "tiktoken-rs"
version = "0.9.1" version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a19830747d9034cd9da43a60eaa8e552dfda7712424aebf187b7a60126bae0d" checksum = "fac4a168cfc1d8ed65bf17a6ee0843ad9a68f863c63c0fb2fa7eab67838782ee"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"base64 0.22.1", "base64 0.22.1",

2
Cargo.toml
View File

@@ -33,7 +33,7 @@ serde = { version = "1.0.*", features = ["derive"], default-features = false }
serde_json = "1.0.*" serde_json = "1.0.*"
serde_yaml_ng = "0.10.*" serde_yaml_ng = "0.10.*"
tempfile = "3.27.*" tempfile = "3.27.*"
tiktoken-rs = { version = "0.9.*", default-features = false } tiktoken-rs = { version = "0.11.*", default-features = false }
tokio = { version = "1.51.*", features = ["rt", "rt-multi-thread", "macros"] } tokio = { version = "1.51.*", features = ["rt", "rt-multi-thread", "macros"] }
tracing = "0.1.*" tracing = "0.1.*"
tracing-subscriber = { version = "0.3.*", features = ["env-filter"] } tracing-subscriber = { version = "0.3.*", features = ["env-filter"] }

32
src/conversation/llm/tokenization.rs
View File

@@ -1,15 +1,15 @@
use tiktoken_rs::CoreBPE; use tiktoken_rs::CoreBPE;
use tiktoken_rs::get_bpe_from_tokenizer; use tiktoken_rs::bpe_for_tokenizer;
use tiktoken_rs::tokenizer; use tiktoken_rs::tokenizer;
use super::{Author, Message, MessageContent}; use super::{Author, Message, MessageContent};
fn get_bpe_for_model(model: &str) -> CoreBPE { fn get_bpe_for_model(model: &str) -> &'static CoreBPE {
let tokenizer = tokenizer::get_tokenizer(model) let tokenizer = tokenizer::get_tokenizer(model)
.or_else(|| tokenizer::get_tokenizer("gpt-4")) .or_else(|| tokenizer::get_tokenizer("gpt-4"))
.unwrap(); .unwrap();
get_bpe_from_tokenizer(tokenizer).unwrap() bpe_for_tokenizer(tokenizer).unwrap()
} }
pub fn shorten_messages_list_to_context_size( pub fn shorten_messages_list_to_context_size(
@@ -26,7 +26,7 @@ pub fn shorten_messages_list_to_context_size(
// We want to retain the prompt in all cases, so we always count it first. // We want to retain the prompt in all cases, so we always count it first.
// We also always reserve enough tokens for the maximum response we expect. // We also always reserve enough tokens for the maximum response we expect.
let mut current_context_length: u32 = if let Some(prompt_message) = prompt_message { let mut current_context_length: u32 = if let Some(prompt_message) = prompt_message {
calculate_token_size_for_message(&bpe, model, prompt_message) calculate_token_size_for_message(bpe, model, prompt_message)
+ max_response_tokens.unwrap_or(0) + max_response_tokens.unwrap_or(0)
} else { } else {
0 0
@@ -37,7 +37,7 @@ pub fn shorten_messages_list_to_context_size(
let mut messages_to_keep: Vec<Message> = Vec::new(); let mut messages_to_keep: Vec<Message> = Vec::new();
for message in messages { for message in messages {
let tokens_for_message = calculate_token_size_for_message(&bpe, model, &message); let tokens_for_message = calculate_token_size_for_message(bpe, model, &message);
if current_context_length + tokens_for_message > max_context_tokens { if current_context_length + tokens_for_message > max_context_tokens {
break; break;
@@ -94,7 +94,7 @@ pub mod test {
timestamp: chrono::Utc::now(), timestamp: chrono::Utc::now(),
}; };
let tokens = super::calculate_token_size_for_message(&bpe, model, &message); let tokens = super::calculate_token_size_for_message(bpe, model, &message);
assert_eq!(8, tokens); assert_eq!(8, tokens);
} }
@@ -117,7 +117,7 @@ pub mod test {
assert_eq!( assert_eq!(
prompt_length, prompt_length,
super::calculate_token_size_for_message(&bpe, model, &prompt) super::calculate_token_size_for_message(bpe, model, &prompt)
); );
let mut conversation_messages = Vec::new(); let mut conversation_messages = Vec::new();
@@ -132,7 +132,7 @@ pub mod test {
assert_eq!( assert_eq!(
first_length, first_length,
super::calculate_token_size_for_message(&bpe, model, &first) super::calculate_token_size_for_message(bpe, model, &first)
); );
conversation_messages.push(first); conversation_messages.push(first);
@@ -147,7 +147,7 @@ pub mod test {
assert_eq!( assert_eq!(
second_length, second_length,
super::calculate_token_size_for_message(&bpe, model, &second) super::calculate_token_size_for_message(bpe, model, &second)
); );
conversation_messages.push(second); conversation_messages.push(second);
@@ -164,7 +164,7 @@ pub mod test {
assert_eq!( assert_eq!(
third_length, third_length,
super::calculate_token_size_for_message(&bpe, model, &third) super::calculate_token_size_for_message(bpe, model, &third)
); );
conversation_messages.push(third.clone()); conversation_messages.push(third.clone());
@@ -181,7 +181,7 @@ pub mod test {
assert_eq!( assert_eq!(
forth_length, forth_length,
super::calculate_token_size_for_message(&bpe, model, &forth) super::calculate_token_size_for_message(bpe, model, &forth)
); );
conversation_messages.push(forth.clone()); conversation_messages.push(forth.clone());
@@ -227,7 +227,7 @@ pub mod test {
assert_eq!( assert_eq!(
prompt_length, prompt_length,
super::calculate_token_size_for_message(&bpe, model, &prompt) super::calculate_token_size_for_message(bpe, model, &prompt)
); );
let mut conversation_messages = Vec::new(); let mut conversation_messages = Vec::new();
@@ -242,7 +242,7 @@ pub mod test {
assert_eq!( assert_eq!(
first_length, first_length,
super::calculate_token_size_for_message(&bpe, model, &first) super::calculate_token_size_for_message(bpe, model, &first)
); );
conversation_messages.push(first); conversation_messages.push(first);
@@ -257,7 +257,7 @@ pub mod test {
assert_eq!( assert_eq!(
second_length, second_length,
super::calculate_token_size_for_message(&bpe, model, &second) super::calculate_token_size_for_message(bpe, model, &second)
); );
conversation_messages.push(second); conversation_messages.push(second);
@@ -274,7 +274,7 @@ pub mod test {
assert_eq!( assert_eq!(
third_length, third_length,
super::calculate_token_size_for_message(&bpe, model, &third) super::calculate_token_size_for_message(bpe, model, &third)
); );
conversation_messages.push(third.clone()); conversation_messages.push(third.clone());
@@ -291,7 +291,7 @@ pub mod test {
assert_eq!( assert_eq!(
forth_length, forth_length,
super::calculate_token_size_for_message(&bpe, model, &forth) super::calculate_token_size_for_message(bpe, model, &forth)
); );
conversation_messages.push(forth.clone()); conversation_messages.push(forth.clone());