Upgrade server dependencies, manage them with govendor

vendor/github.com/blevesearch/bleve/document/document.go

@@ -1,22 +1,26 @@
 //  Copyright (c) 2014 Couchbase, Inc.
-//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
-//  except in compliance with the License. You may obtain a copy of the License at
-//    http://www.apache.org/licenses/LICENSE-2.0
-//  Unless required by applicable law or agreed to in writing, software distributed under the
-//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-//  either express or implied. See the License for the specific language governing permissions
-//  and limitations under the License.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 package document
 
-import (
-	"fmt"
-)
+import "fmt"
 
 type Document struct {
 	ID              string  `json:"id"`
 	Fields          []Field `json:"fields"`
 	CompositeFields []*CompositeField
+	Number          uint64 `json:"-"`
 }
 
 func NewDocument(id string) *Document {
@@ -54,3 +58,18 @@ func (d *Document) GoString() string {
 	}
 	return fmt.Sprintf("&document.Document{ID:%s, Fields: %s, CompositeFields: %s}", d.ID, fields, compositeFields)
 }
+
+func (d *Document) NumPlainTextBytes() uint64 {
+	rv := uint64(0)
+	for _, field := range d.Fields {
+		rv += field.NumPlainTextBytes()
+	}
+	for _, compositeField := range d.CompositeFields {
+		for _, field := range d.Fields {
+			if compositeField.includesField(field.Name()) {
+				rv += field.NumPlainTextBytes()
+			}
+		}
+	}
+	return rv
+}

vendor/github.com/blevesearch/bleve/document/field.go

@@ -1,11 +1,16 @@
 //  Copyright (c) 2014 Couchbase, Inc.
-//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
-//  except in compliance with the License. You may obtain a copy of the License at
-//    http://www.apache.org/licenses/LICENSE-2.0
-//  Unless required by applicable law or agreed to in writing, software distributed under the
-//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-//  either express or implied. See the License for the specific language governing permissions
-//  and limitations under the License.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 package document
 
@@ -14,9 +19,21 @@ import (
 )
 
 type Field interface {
+	// Name returns the path of the field from the root DocumentMapping.
+	// A root field path is "field", a subdocument field is "parent.field".
 	Name() string
+	// ArrayPositions returns the intermediate document and field indices
+	// required to resolve the field value in the document. For example, if the
+	// field path is "doc1.doc2.field" where doc1 and doc2 are slices or
+	// arrays, ArrayPositions returns 2 indices used to resolve "doc2" value in
+	// "doc1", then "field" in "doc2".
 	ArrayPositions() []uint64
 	Options() IndexingOptions
 	Analyze() (int, analysis.TokenFrequencies)
 	Value() []byte
+
+	// NumPlainTextBytes should return the number of plain text bytes
+	// that this field represents - this is a common metric for tracking
+	// the rate of indexing
+	NumPlainTextBytes() uint64
 }

vendor/github.com/blevesearch/bleve/document/field_boolean.go

@@ -0,0 +1,107 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+	"fmt"
+
+	"github.com/blevesearch/bleve/analysis"
+)
+
+const DefaultBooleanIndexingOptions = StoreField | IndexField
+
+type BooleanField struct {
+	name              string
+	arrayPositions    []uint64
+	options           IndexingOptions
+	value             []byte
+	numPlainTextBytes uint64
+}
+
+func (b *BooleanField) Name() string {
+	return b.name
+}
+
+func (b *BooleanField) ArrayPositions() []uint64 {
+	return b.arrayPositions
+}
+
+func (b *BooleanField) Options() IndexingOptions {
+	return b.options
+}
+
+func (b *BooleanField) Analyze() (int, analysis.TokenFrequencies) {
+	tokens := make(analysis.TokenStream, 0)
+	tokens = append(tokens, &analysis.Token{
+		Start:    0,
+		End:      len(b.value),
+		Term:     b.value,
+		Position: 1,
+		Type:     analysis.Boolean,
+	})
+
+	fieldLength := len(tokens)
+	tokenFreqs := analysis.TokenFrequency(tokens, b.arrayPositions, b.options.IncludeTermVectors())
+	return fieldLength, tokenFreqs
+}
+
+func (b *BooleanField) Value() []byte {
+	return b.value
+}
+
+func (b *BooleanField) Boolean() (bool, error) {
+	if len(b.value) == 1 {
+		return b.value[0] == 'T', nil
+	}
+	return false, fmt.Errorf("boolean field has %d bytes", len(b.value))
+}
+
+func (b *BooleanField) GoString() string {
+	return fmt.Sprintf("&document.BooleanField{Name:%s, Options: %s, Value: %s}", b.name, b.options, b.value)
+}
+
+func (b *BooleanField) NumPlainTextBytes() uint64 {
+	return b.numPlainTextBytes
+}
+
+func NewBooleanFieldFromBytes(name string, arrayPositions []uint64, value []byte) *BooleanField {
+	return &BooleanField{
+		name:              name,
+		arrayPositions:    arrayPositions,
+		value:             value,
+		options:           DefaultNumericIndexingOptions,
+		numPlainTextBytes: uint64(len(value)),
+	}
+}
+
+func NewBooleanField(name string, arrayPositions []uint64, b bool) *BooleanField {
+	return NewBooleanFieldWithIndexingOptions(name, arrayPositions, b, DefaultNumericIndexingOptions)
+}
+
+func NewBooleanFieldWithIndexingOptions(name string, arrayPositions []uint64, b bool, options IndexingOptions) *BooleanField {
+	numPlainTextBytes := 5
+	v := []byte("F")
+	if b {
+		numPlainTextBytes = 4
+		v = []byte("T")
+	}
+	return &BooleanField{
+		name:              name,
+		arrayPositions:    arrayPositions,
+		value:             v,
+		options:           options,
+		numPlainTextBytes: uint64(numPlainTextBytes),
+	}
+}

vendor/github.com/blevesearch/bleve/document/field_composite.go

@@ -1,11 +1,16 @@
 //  Copyright (c) 2014 Couchbase, Inc.
-//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
-//  except in compliance with the License. You may obtain a copy of the License at
-//    http://www.apache.org/licenses/LICENSE-2.0
-//  Unless required by applicable law or agreed to in writing, software distributed under the
-//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-//  either express or implied. See the License for the specific language governing permissions
-//  and limitations under the License.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 package document
 
@@ -31,11 +36,12 @@ func NewCompositeField(name string, defaultInclude bool, include []string, exclu
 
 func NewCompositeFieldWithIndexingOptions(name string, defaultInclude bool, include []string, exclude []string, options IndexingOptions) *CompositeField {
 	rv := &CompositeField{
-		name:           name,
-		options:        options,
-		defaultInclude: defaultInclude,
-		includedFields: make(map[string]bool, len(include)),
-		excludedFields: make(map[string]bool, len(exclude)),
+		name:                 name,
+		options:              options,
+		defaultInclude:       defaultInclude,
+		includedFields:       make(map[string]bool, len(include)),
+		excludedFields:       make(map[string]bool, len(exclude)),
+		compositeFrequencies: make(analysis.TokenFrequencies),
 	}
 
 	for _, i := range include {
@@ -68,7 +74,11 @@ func (c *CompositeField) Value() []byte {
 	return []byte{}
 }
 
-func (c *CompositeField) Compose(field string, length int, freq analysis.TokenFrequencies) {
+func (c *CompositeField) NumPlainTextBytes() uint64 {
+	return 0
+}
+
+func (c *CompositeField) includesField(field string) bool {
 	shouldInclude := c.defaultInclude
 	_, fieldShouldBeIncluded := c.includedFields[field]
 	if fieldShouldBeIncluded {
@@ -78,9 +88,12 @@ func (c *CompositeField) Compose(field string, length int, freq analysis.TokenFr
 	if fieldShouldBeExcluded {
 		shouldInclude = false
 	}
+	return shouldInclude
+}
 
-	if shouldInclude {
+func (c *CompositeField) Compose(field string, length int, freq analysis.TokenFrequencies) {
+	if c.includesField(field) {
 		c.totalLength += length
-		c.compositeFrequencies = c.compositeFrequencies.MergeAll(field, freq)
+		c.compositeFrequencies.MergeAll(field, freq)
 	}
 }

vendor/github.com/blevesearch/bleve/document/field_datetime.go

@@ -1,11 +1,16 @@
 //  Copyright (c) 2014 Couchbase, Inc.
-//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
-//  except in compliance with the License. You may obtain a copy of the License at
-//    http://www.apache.org/licenses/LICENSE-2.0
-//  Unless required by applicable law or agreed to in writing, software distributed under the
-//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-//  either express or implied. See the License for the specific language governing permissions
-//  and limitations under the License.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 package document
 
@@ -15,7 +20,7 @@ import (
 	"time"
 
 	"github.com/blevesearch/bleve/analysis"
-	"github.com/blevesearch/bleve/numeric_util"
+	"github.com/blevesearch/bleve/numeric"
 )
 
 const DefaultDateTimeIndexingOptions = StoreField | IndexField
@@ -25,10 +30,11 @@ var MinTimeRepresentable = time.Unix(0, math.MinInt64)
 var MaxTimeRepresentable = time.Unix(0, math.MaxInt64)
 
 type DateTimeField struct {
-	name           string
-	arrayPositions []uint64
-	options        IndexingOptions
-	value          numeric_util.PrefixCoded
+	name              string
+	arrayPositions    []uint64
+	options           IndexingOptions
+	value             numeric.PrefixCoded
+	numPlainTextBytes uint64
 }
 
 func (n *DateTimeField) Name() string {
@@ -58,7 +64,7 @@ func (n *DateTimeField) Analyze() (int, analysis.TokenFrequencies) {
 
 		shift := DefaultDateTimePrecisionStep
 		for shift < 64 {
-			shiftEncoded, err := numeric_util.NewPrefixCodedInt64(original, shift)
+			shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift)
 			if err != nil {
 				break
 			}
@@ -75,7 +81,7 @@ func (n *DateTimeField) Analyze() (int, analysis.TokenFrequencies) {
 	}
 
 	fieldLength := len(tokens)
-	tokenFreqs := analysis.TokenFrequency(tokens)
+	tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors())
 	return fieldLength, tokenFreqs
 }
 
@@ -95,12 +101,17 @@ func (n *DateTimeField) GoString() string {
 	return fmt.Sprintf("&document.DateField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value)
 }
 
+func (n *DateTimeField) NumPlainTextBytes() uint64 {
+	return n.numPlainTextBytes
+}
+
 func NewDateTimeFieldFromBytes(name string, arrayPositions []uint64, value []byte) *DateTimeField {
 	return &DateTimeField{
-		name:           name,
-		arrayPositions: arrayPositions,
-		value:          value,
-		options:        DefaultDateTimeIndexingOptions,
+		name:              name,
+		arrayPositions:    arrayPositions,
+		value:             value,
+		options:           DefaultDateTimeIndexingOptions,
+		numPlainTextBytes: uint64(len(value)),
 	}
 }
 
@@ -111,12 +122,15 @@ func NewDateTimeField(name string, arrayPositions []uint64, dt time.Time) (*Date
 func NewDateTimeFieldWithIndexingOptions(name string, arrayPositions []uint64, dt time.Time, options IndexingOptions) (*DateTimeField, error) {
 	if canRepresent(dt) {
 		dtInt64 := dt.UnixNano()
-		prefixCoded := numeric_util.MustNewPrefixCodedInt64(dtInt64, 0)
+		prefixCoded := numeric.MustNewPrefixCodedInt64(dtInt64, 0)
 		return &DateTimeField{
 			name:           name,
 			arrayPositions: arrayPositions,
 			value:          prefixCoded,
 			options:        options,
+			// not correct, just a place holder until we revisit how fields are
+			// represented and can fix this better
+			numPlainTextBytes: uint64(8),
 		}, nil
 	}
 	return nil, fmt.Errorf("cannot represent %s in this type", dt)

vendor/github.com/blevesearch/bleve/document/field_geopoint.go

@@ -0,0 +1,137 @@
+//  Copyright (c) 2017 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+	"fmt"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/geo"
+	"github.com/blevesearch/bleve/numeric"
+)
+
+var GeoPrecisionStep uint = 9
+
+type GeoPointField struct {
+	name              string
+	arrayPositions    []uint64
+	options           IndexingOptions
+	value             numeric.PrefixCoded
+	numPlainTextBytes uint64
+}
+
+func (n *GeoPointField) Name() string {
+	return n.name
+}
+
+func (n *GeoPointField) ArrayPositions() []uint64 {
+	return n.arrayPositions
+}
+
+func (n *GeoPointField) Options() IndexingOptions {
+	return n.options
+}
+
+func (n *GeoPointField) Analyze() (int, analysis.TokenFrequencies) {
+	tokens := make(analysis.TokenStream, 0)
+	tokens = append(tokens, &analysis.Token{
+		Start:    0,
+		End:      len(n.value),
+		Term:     n.value,
+		Position: 1,
+		Type:     analysis.Numeric,
+	})
+
+	original, err := n.value.Int64()
+	if err == nil {
+
+		shift := GeoPrecisionStep
+		for shift < 64 {
+			shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift)
+			if err != nil {
+				break
+			}
+			token := analysis.Token{
+				Start:    0,
+				End:      len(shiftEncoded),
+				Term:     shiftEncoded,
+				Position: 1,
+				Type:     analysis.Numeric,
+			}
+			tokens = append(tokens, &token)
+			shift += GeoPrecisionStep
+		}
+	}
+
+	fieldLength := len(tokens)
+	tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors())
+	return fieldLength, tokenFreqs
+}
+
+func (n *GeoPointField) Value() []byte {
+	return n.value
+}
+
+func (n *GeoPointField) Lon() (float64, error) {
+	i64, err := n.value.Int64()
+	if err != nil {
+		return 0.0, err
+	}
+	return geo.MortonUnhashLon(uint64(i64)), nil
+}
+
+func (n *GeoPointField) Lat() (float64, error) {
+	i64, err := n.value.Int64()
+	if err != nil {
+		return 0.0, err
+	}
+	return geo.MortonUnhashLat(uint64(i64)), nil
+}
+
+func (n *GeoPointField) GoString() string {
+	return fmt.Sprintf("&document.GeoPointField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value)
+}
+
+func (n *GeoPointField) NumPlainTextBytes() uint64 {
+	return n.numPlainTextBytes
+}
+
+func NewGeoPointFieldFromBytes(name string, arrayPositions []uint64, value []byte) *GeoPointField {
+	return &GeoPointField{
+		name:              name,
+		arrayPositions:    arrayPositions,
+		value:             value,
+		options:           DefaultNumericIndexingOptions,
+		numPlainTextBytes: uint64(len(value)),
+	}
+}
+
+func NewGeoPointField(name string, arrayPositions []uint64, lon, lat float64) *GeoPointField {
+	return NewGeoPointFieldWithIndexingOptions(name, arrayPositions, lon, lat, DefaultNumericIndexingOptions)
+}
+
+func NewGeoPointFieldWithIndexingOptions(name string, arrayPositions []uint64, lon, lat float64, options IndexingOptions) *GeoPointField {
+	mhash := geo.MortonHash(lon, lat)
+	prefixCoded := numeric.MustNewPrefixCodedInt64(int64(mhash), 0)
+	return &GeoPointField{
+		name:           name,
+		arrayPositions: arrayPositions,
+		value:          prefixCoded,
+		options:        options,
+		// not correct, just a place holder until we revisit how fields are
+		// represented and can fix this better
+		numPlainTextBytes: uint64(8),
+	}
+}

vendor/github.com/blevesearch/bleve/document/field_numeric.go

@@ -1,11 +1,16 @@
 //  Copyright (c) 2014 Couchbase, Inc.
-//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
-//  except in compliance with the License. You may obtain a copy of the License at
-//    http://www.apache.org/licenses/LICENSE-2.0
-//  Unless required by applicable law or agreed to in writing, software distributed under the
-//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-//  either express or implied. See the License for the specific language governing permissions
-//  and limitations under the License.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 package document
 
@@ -13,7 +18,7 @@ import (
 	"fmt"
 
 	"github.com/blevesearch/bleve/analysis"
-	"github.com/blevesearch/bleve/numeric_util"
+	"github.com/blevesearch/bleve/numeric"
 )
 
 const DefaultNumericIndexingOptions = StoreField | IndexField
@@ -21,10 +26,11 @@ const DefaultNumericIndexingOptions = StoreField | IndexField
 const DefaultPrecisionStep uint = 4
 
 type NumericField struct {
-	name           string
-	arrayPositions []uint64
-	options        IndexingOptions
-	value          numeric_util.PrefixCoded
+	name              string
+	arrayPositions    []uint64
+	options           IndexingOptions
+	value             numeric.PrefixCoded
+	numPlainTextBytes uint64
 }
 
 func (n *NumericField) Name() string {
@@ -54,7 +60,7 @@ func (n *NumericField) Analyze() (int, analysis.TokenFrequencies) {
 
 		shift := DefaultPrecisionStep
 		for shift < 64 {
-			shiftEncoded, err := numeric_util.NewPrefixCodedInt64(original, shift)
+			shiftEncoded, err := numeric.NewPrefixCodedInt64(original, shift)
 			if err != nil {
 				break
 			}
@@ -71,7 +77,7 @@ func (n *NumericField) Analyze() (int, analysis.TokenFrequencies) {
 	}
 
 	fieldLength := len(tokens)
-	tokenFreqs := analysis.TokenFrequency(tokens)
+	tokenFreqs := analysis.TokenFrequency(tokens, n.arrayPositions, n.options.IncludeTermVectors())
 	return fieldLength, tokenFreqs
 }
 
@@ -84,19 +90,24 @@ func (n *NumericField) Number() (float64, error) {
 	if err != nil {
 		return 0.0, err
 	}
-	return numeric_util.Int64ToFloat64(i64), nil
+	return numeric.Int64ToFloat64(i64), nil
 }
 
 func (n *NumericField) GoString() string {
 	return fmt.Sprintf("&document.NumericField{Name:%s, Options: %s, Value: %s}", n.name, n.options, n.value)
 }
 
+func (n *NumericField) NumPlainTextBytes() uint64 {
+	return n.numPlainTextBytes
+}
+
 func NewNumericFieldFromBytes(name string, arrayPositions []uint64, value []byte) *NumericField {
 	return &NumericField{
-		name:           name,
-		arrayPositions: arrayPositions,
-		value:          value,
-		options:        DefaultNumericIndexingOptions,
+		name:              name,
+		arrayPositions:    arrayPositions,
+		value:             value,
+		options:           DefaultNumericIndexingOptions,
+		numPlainTextBytes: uint64(len(value)),
 	}
 }
 
@@ -105,12 +116,15 @@ func NewNumericField(name string, arrayPositions []uint64, number float64) *Nume
 }
 
 func NewNumericFieldWithIndexingOptions(name string, arrayPositions []uint64, number float64, options IndexingOptions) *NumericField {
-	numberInt64 := numeric_util.Float64ToInt64(number)
-	prefixCoded := numeric_util.MustNewPrefixCodedInt64(numberInt64, 0)
+	numberInt64 := numeric.Float64ToInt64(number)
+	prefixCoded := numeric.MustNewPrefixCodedInt64(numberInt64, 0)
 	return &NumericField{
 		name:           name,
 		arrayPositions: arrayPositions,
 		value:          prefixCoded,
 		options:        options,
+		// not correct, just a place holder until we revisit how fields are
+		// represented and can fix this better
+		numPlainTextBytes: uint64(8),
 	}
 }

vendor/github.com/blevesearch/bleve/document/field_numeric_test.go

@@ -1,25 +0,0 @@
-//  Copyright (c) 2014 Couchbase, Inc.
-//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
-//  except in compliance with the License. You may obtain a copy of the License at
-//    http://www.apache.org/licenses/LICENSE-2.0
-//  Unless required by applicable law or agreed to in writing, software distributed under the
-//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-//  either express or implied. See the License for the specific language governing permissions
-//  and limitations under the License.
-
-package document
-
-import (
-	"testing"
-)
-
-func TestNumericField(t *testing.T) {
-	nf := NewNumericField("age", []uint64{}, 3.4)
-	numTokens, tokenFreqs := nf.Analyze()
-	if numTokens != 16 {
-		t.Errorf("expected 16 tokens")
-	}
-	if len(tokenFreqs) != 16 {
-		t.Errorf("expected 16 token freqs")
-	}
-}

vendor/github.com/blevesearch/bleve/document/field_text.go

@@ -1,11 +1,16 @@
 //  Copyright (c) 2014 Couchbase, Inc.
-//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
-//  except in compliance with the License. You may obtain a copy of the License at
-//    http://www.apache.org/licenses/LICENSE-2.0
-//  Unless required by applicable law or agreed to in writing, software distributed under the
-//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-//  either express or implied. See the License for the specific language governing permissions
-//  and limitations under the License.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 package document
 
@@ -18,11 +23,12 @@ import (
 const DefaultTextIndexingOptions = IndexField
 
 type TextField struct {
-	name           string
-	arrayPositions []uint64
-	options        IndexingOptions
-	analyzer       *analysis.Analyzer
-	value          []byte
+	name              string
+	arrayPositions    []uint64
+	options           IndexingOptions
+	analyzer          *analysis.Analyzer
+	value             []byte
+	numPlainTextBytes uint64
 }
 
 func (t *TextField) Name() string {
@@ -60,7 +66,7 @@ func (t *TextField) Analyze() (int, analysis.TokenFrequencies) {
 		}
 	}
 	fieldLength := len(tokens) // number of tokens in this doc field
-	tokenFreqs := analysis.TokenFrequency(tokens)
+	tokenFreqs := analysis.TokenFrequency(tokens, t.arrayPositions, t.options.IncludeTermVectors())
 	return fieldLength, tokenFreqs
 }
 
@@ -69,7 +75,11 @@ func (t *TextField) Value() []byte {
 }
 
 func (t *TextField) GoString() string {
-	return fmt.Sprintf("&document.TextField{Name:%s, Options: %s, Analyzer: %s, Value: %s}", t.name, t.options, t.analyzer, t.value)
+	return fmt.Sprintf("&document.TextField{Name:%s, Options: %s, Analyzer: %v, Value: %s, ArrayPositions: %v}", t.name, t.options, t.analyzer, t.value, t.arrayPositions)
+}
+
+func (t *TextField) NumPlainTextBytes() uint64 {
+	return t.numPlainTextBytes
 }
 
 func NewTextField(name string, arrayPositions []uint64, value []byte) *TextField {
@@ -78,29 +88,32 @@ func NewTextField(name string, arrayPositions []uint64, value []byte) *TextField
 
 func NewTextFieldWithIndexingOptions(name string, arrayPositions []uint64, value []byte, options IndexingOptions) *TextField {
 	return &TextField{
-		name:           name,
-		arrayPositions: arrayPositions,
-		options:        options,
-		value:          value,
+		name:              name,
+		arrayPositions:    arrayPositions,
+		options:           options,
+		value:             value,
+		numPlainTextBytes: uint64(len(value)),
 	}
 }
 
 func NewTextFieldWithAnalyzer(name string, arrayPositions []uint64, value []byte, analyzer *analysis.Analyzer) *TextField {
 	return &TextField{
-		name:           name,
-		arrayPositions: arrayPositions,
-		options:        DefaultTextIndexingOptions,
-		analyzer:       analyzer,
-		value:          value,
+		name:              name,
+		arrayPositions:    arrayPositions,
+		options:           DefaultTextIndexingOptions,
+		analyzer:          analyzer,
+		value:             value,
+		numPlainTextBytes: uint64(len(value)),
 	}
 }
 
 func NewTextFieldCustom(name string, arrayPositions []uint64, value []byte, options IndexingOptions, analyzer *analysis.Analyzer) *TextField {
 	return &TextField{
-		name:           name,
-		arrayPositions: arrayPositions,
-		options:        options,
-		analyzer:       analyzer,
-		value:          value,
+		name:              name,
+		arrayPositions:    arrayPositions,
+		options:           options,
+		analyzer:          analyzer,
+		value:             value,
+		numPlainTextBytes: uint64(len(value)),
 	}
 }

vendor/github.com/blevesearch/bleve/document/indexing_options.go

@@ -1,11 +1,16 @@
 //  Copyright (c) 2014 Couchbase, Inc.
-//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
-//  except in compliance with the License. You may obtain a copy of the License at
-//    http://www.apache.org/licenses/LICENSE-2.0
-//  Unless required by applicable law or agreed to in writing, software distributed under the
-//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-//  either express or implied. See the License for the specific language governing permissions
-//  and limitations under the License.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
 
 package document
 

vendor/github.com/blevesearch/bleve/document/indexing_options_test.go

@@ -1,69 +0,0 @@
-//  Copyright (c) 2014 Couchbase, Inc.
-//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
-//  except in compliance with the License. You may obtain a copy of the License at
-//    http://www.apache.org/licenses/LICENSE-2.0
-//  Unless required by applicable law or agreed to in writing, software distributed under the
-//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-//  either express or implied. See the License for the specific language governing permissions
-//  and limitations under the License.
-
-package document
-
-import (
-	"testing"
-)
-
-func TestIndexingOptions(t *testing.T) {
-	tests := []struct {
-		options            IndexingOptions
-		isIndexed          bool
-		isStored           bool
-		includeTermVectors bool
-	}{
-		{
-			options:            IndexField | StoreField | IncludeTermVectors,
-			isIndexed:          true,
-			isStored:           true,
-			includeTermVectors: true,
-		},
-		{
-			options:            IndexField | IncludeTermVectors,
-			isIndexed:          true,
-			isStored:           false,
-			includeTermVectors: true,
-		},
-		{
-			options:            StoreField | IncludeTermVectors,
-			isIndexed:          false,
-			isStored:           true,
-			includeTermVectors: true,
-		},
-		{
-			options:            IndexField,
-			isIndexed:          true,
-			isStored:           false,
-			includeTermVectors: false,
-		},
-		{
-			options:            StoreField,
-			isIndexed:          false,
-			isStored:           true,
-			includeTermVectors: false,
-		},
-	}
-
-	for _, test := range tests {
-		actuallyIndexed := test.options.IsIndexed()
-		if actuallyIndexed != test.isIndexed {
-			t.Errorf("expected indexed to be %v, got %v for %d", test.isIndexed, actuallyIndexed, test.options)
-		}
-		actuallyStored := test.options.IsStored()
-		if actuallyStored != test.isStored {
-			t.Errorf("expected stored to be %v, got %v for %d", test.isStored, actuallyStored, test.options)
-		}
-		actuallyIncludeTermVectors := test.options.IncludeTermVectors()
-		if actuallyIncludeTermVectors != test.includeTermVectors {
-			t.Errorf("expected includeTermVectors to be %v, got %v for %d", test.includeTermVectors, actuallyIncludeTermVectors, test.options)
-		}
-	}
-}