21 lines
415 B
Go
21 lines
415 B
Go
|
// Copyright 2013 The Go Authors. All rights reserved.
|
||
|
|
// Use of this source code is governed by a BSD-style
|
||
|
|
// license that can be found in the LICENSE file.
|
||
|
|
|
||
|
|
package segment
|
||
|
|
|
||
|
|
// Exported for testing only.
|
||
|
|
import (
|
||
|
|
"unicode/utf8"
|
||
|
|
)
|
||
|
|
|
||
|
|
func (s *Segmenter) MaxTokenSize(n int) {
|
||
|
|
if n < utf8.UTFMax || n > 1e9 {
|
||
|
|
panic("bad max token size")
|
||
|
|
}
|
||
|
|
if n < len(s.buf) {
|
||
|
|
s.buf = make([]byte, n)
|
||
|
|
}
|
||
|
|
s.maxTokenSize = n
|
||
|
|
}
|