diff --git a/.buckconfig b/.buckconfig index e04d6b95ce7..603fa928bcb 100644 --- a/.buckconfig +++ b/.buckconfig @@ -3,6 +3,7 @@ [buildfile] name = TARGETS + name_v2 = TARGETS,BUCK [repositories] root = . diff --git a/extension/llm/runner/test/test_text_llm_runner.cpp b/extension/llm/runner/test/test_text_llm_runner.cpp index ae56abd3d97..c9b57fb7391 100644 --- a/extension/llm/runner/test/test_text_llm_runner.cpp +++ b/extension/llm/runner/test/test_text_llm_runner.cpp @@ -43,6 +43,11 @@ class MockTokenizer : public ::tokenizers::Tokenizer { decode, (uint64_t, uint64_t), (const)); + MOCK_METHOD( + ::tokenizers::Result, + id_to_piece, + (uint64_t), + (const)); MOCK_METHOD(uint64_t, bos_tok, (), (const)); MOCK_METHOD(uint64_t, eos_tok, (), (const)); MOCK_METHOD(uint64_t, vocab_size, (), (const)); @@ -127,6 +132,10 @@ class RunnerTest : public Test { return ::tokenizers::Result("token"); }); + ON_CALL(*tokenizer, id_to_piece).WillByDefault([](uint64_t) { + return ::tokenizers::Result("piece"); + }); + ON_CALL(*tokenizer, bos_tok()).WillByDefault(Return(1)); ON_CALL(*tokenizer, eos_tok()).WillByDefault(Return(2)); ON_CALL(*tokenizer, vocab_size()).WillByDefault(Return(100)); diff --git a/extension/llm/tokenizers b/extension/llm/tokenizers index 3aada3fe28c..37e1c7ed13f 160000 --- a/extension/llm/tokenizers +++ b/extension/llm/tokenizers @@ -1 +1 @@ -Subproject commit 3aada3fe28c945d14d5ec62254eb56ccdf10eb11 +Subproject commit 37e1c7ed13fa04accd696c776c2f05b4b12fe61d