mirror of
https://github.com/ollama/ollama.git
synced 2026-03-31 03:38:42 +07:00
97323d1c68
This change adds a new x/tokenizer package which includes: * New BPE and SentencePiece tokenizers * Removing the dependency on the imagegen tokenizers * Fixes to multibyte decoding in the pipeline * Various correctness and benchmark tests Not included in this PR is the WordPiece tokenizer for BERT models which will be added when we add embedding models. The imagegen tokenizers will also be removed in a follow-up PR.
47 lines
1.0 KiB
Go
47 lines
1.0 KiB
Go
package mlxrunner
|
|
|
|
import (
|
|
"bytes"
|
|
"testing"
|
|
)
|
|
|
|
func TestFlushValidUTF8Prefix_PreservesIncompleteRune(t *testing.T) {
|
|
var b bytes.Buffer
|
|
|
|
b.Write([]byte{0xE3, 0x81})
|
|
if got := flushValidUTF8Prefix(&b); got != "" {
|
|
t.Fatalf("first flush = %q, want empty", got)
|
|
}
|
|
|
|
b.Write([]byte{0x93, 0xE3})
|
|
if got := flushValidUTF8Prefix(&b); got != "こ" {
|
|
t.Fatalf("second flush = %q, want %q", got, "こ")
|
|
}
|
|
|
|
if got := b.Bytes(); !bytes.Equal(got, []byte{0xE3}) {
|
|
t.Fatalf("buffer after second flush = %v, want %v", got, []byte{0xE3})
|
|
}
|
|
|
|
b.Write([]byte{0x82, 0x93})
|
|
if got := flushValidUTF8Prefix(&b); got != "ん" {
|
|
t.Fatalf("third flush = %q, want %q", got, "ん")
|
|
}
|
|
|
|
if b.Len() != 0 {
|
|
t.Fatalf("buffer not empty after third flush: %d", b.Len())
|
|
}
|
|
}
|
|
|
|
func TestFlushValidUTF8Prefix_ValidText(t *testing.T) {
|
|
var b bytes.Buffer
|
|
b.WriteString("hello 世界")
|
|
|
|
if got := flushValidUTF8Prefix(&b); got != "hello 世界" {
|
|
t.Fatalf("flush = %q, want %q", got, "hello 世界")
|
|
}
|
|
|
|
if b.Len() != 0 {
|
|
t.Fatalf("buffer not empty after flush: %d", b.Len())
|
|
}
|
|
}
|