Switch from Godep to go vendoring

This commit is contained in:
Ken-Håvard Lieng 2016-03-01 01:51:26 +01:00
parent 6b37713bc0
commit cd317761c5
1504 changed files with 263076 additions and 34441 deletions

View file

@ -0,0 +1,138 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package simple
import (
"unicode/utf8"
"github.com/blevesearch/bleve/registry"
"github.com/blevesearch/bleve/search/highlight"
)
const Name = "simple"
const defaultFragmentSize = 200
type Fragmenter struct {
fragmentSize int
}
func NewFragmenter(fragmentSize int) *Fragmenter {
return &Fragmenter{
fragmentSize: fragmentSize,
}
}
func (s *Fragmenter) Fragment(orig []byte, ot highlight.TermLocations) []*highlight.Fragment {
rv := make([]*highlight.Fragment, 0)
maxbegin := 0
OUTER:
for currTermIndex, termLocation := range ot {
// start with this
// it should be the highest scoring fragment with this term first
start := termLocation.Start
end := start
used := 0
for end < len(orig) && used < s.fragmentSize {
r, size := utf8.DecodeRune(orig[end:])
if r == utf8.RuneError {
continue OUTER // bail
}
end += size
used += 1
}
// if we still have more characters available to us
// push back towards begining
// without cross maxbegin
for start > 0 && used < s.fragmentSize {
r, size := utf8.DecodeLastRune(orig[0:start])
if r == utf8.RuneError {
continue OUTER // bail
}
if start-size >= maxbegin {
start -= size
used += 1
} else {
break
}
}
// however, we'd rather have the tokens centered more in the frag
// lets try to do that as best we can, without affecting the score
// find the end of the last term in this fragment
minend := end
for _, innerTermLocation := range ot[currTermIndex:] {
if innerTermLocation.End > end {
break
}
minend = innerTermLocation.End
}
// find the smaller of the two rooms to move
roomToMove := utf8.RuneCount(orig[minend:end])
roomToMoveStart := 0
if start >= maxbegin {
roomToMoveStart = utf8.RuneCount(orig[maxbegin:start])
}
if roomToMoveStart < roomToMove {
roomToMove = roomToMoveStart
}
offset := roomToMove / 2
for offset > 0 {
r, size := utf8.DecodeLastRune(orig[0:start])
if r == utf8.RuneError {
continue OUTER // bail
}
start -= size
r, size = utf8.DecodeLastRune(orig[0:end])
if r == utf8.RuneError {
continue OUTER // bail
}
end -= size
offset--
}
rv = append(rv, &highlight.Fragment{Orig: orig, Start: start - offset, End: end - offset})
// set maxbegin to the end of the current term location
// so that next one won't back up to include it
maxbegin = termLocation.End
}
if len(ot) == 0 {
// if there were no terms to highlight
// produce a single fragment from the beginning
start := 0
end := start + s.fragmentSize
if end > len(orig) {
end = len(orig)
}
rv = append(rv, &highlight.Fragment{Orig: orig, Start: start, End: end})
}
return rv
}
func Constructor(config map[string]interface{}, cache *registry.Cache) (highlight.Fragmenter, error) {
size := defaultFragmentSize
sizeVal, ok := config["size"].(float64)
if ok {
size = int(sizeVal)
}
return NewFragmenter(size), nil
}
func init() {
registry.RegisterFragmenter(Name, Constructor)
}

View file

@ -0,0 +1,295 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package simple
import (
"reflect"
"testing"
"github.com/blevesearch/bleve/search/highlight"
)
func TestSimpleFragmenter(t *testing.T) {
tests := []struct {
orig []byte
fragments []*highlight.Fragment
ot highlight.TermLocations
size int
}{
{
orig: []byte("this is a test"),
fragments: []*highlight.Fragment{
&highlight.Fragment{
Orig: []byte("this is a test"),
Start: 0,
End: 14,
},
},
ot: highlight.TermLocations{
&highlight.TermLocation{
Term: "test",
Pos: 4,
Start: 10,
End: 14,
},
},
size: 100,
},
{
orig: []byte("0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"),
fragments: []*highlight.Fragment{
&highlight.Fragment{
Orig: []byte("0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"),
Start: 0,
End: 100,
},
},
ot: highlight.TermLocations{
&highlight.TermLocation{
Term: "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789",
Pos: 1,
Start: 0,
End: 100,
},
},
size: 100,
},
{
orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
fragments: []*highlight.Fragment{
&highlight.Fragment{
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
Start: 0,
End: 100,
},
&highlight.Fragment{
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
Start: 10,
End: 101,
},
&highlight.Fragment{
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
Start: 20,
End: 101,
},
&highlight.Fragment{
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
Start: 30,
End: 101,
},
&highlight.Fragment{
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
Start: 40,
End: 101,
},
&highlight.Fragment{
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
Start: 50,
End: 101,
},
&highlight.Fragment{
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
Start: 60,
End: 101,
},
&highlight.Fragment{
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
Start: 70,
End: 101,
},
&highlight.Fragment{
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
Start: 80,
End: 101,
},
&highlight.Fragment{
Orig: []byte("01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"),
Start: 90,
End: 101,
},
},
ot: highlight.TermLocations{
&highlight.TermLocation{
Term: "0123456789",
Pos: 1,
Start: 0,
End: 10,
},
&highlight.TermLocation{
Term: "0123456789",
Pos: 2,
Start: 10,
End: 20,
},
&highlight.TermLocation{
Term: "0123456789",
Pos: 3,
Start: 20,
End: 30,
},
&highlight.TermLocation{
Term: "0123456789",
Pos: 4,
Start: 30,
End: 40,
},
&highlight.TermLocation{
Term: "0123456789",
Pos: 5,
Start: 40,
End: 50,
},
&highlight.TermLocation{
Term: "0123456789",
Pos: 6,
Start: 50,
End: 60,
},
&highlight.TermLocation{
Term: "0123456789",
Pos: 7,
Start: 60,
End: 70,
},
&highlight.TermLocation{
Term: "0123456789",
Pos: 8,
Start: 70,
End: 80,
},
&highlight.TermLocation{
Term: "0123456789",
Pos: 9,
Start: 80,
End: 90,
},
&highlight.TermLocation{
Term: "0123456789",
Pos: 10,
Start: 90,
End: 100,
},
},
size: 100,
},
{
orig: []byte("[[पानी का स्वाद]] [[नीलेश रघुवंशी]] का कविता संग्रह हैं। इस कृति के लिए उन्हें २००४ में [[केदार सम्मान]] से सम्मानित किया गया है।{{केदार सम्मान से सम्मानित कृतियाँ}}"),
fragments: []*highlight.Fragment{
&highlight.Fragment{
Orig: []byte("[[पानी का स्वाद]] [[नीलेश रघुवंशी]] का कविता संग्रह हैं। इस कृति के लिए उन्हें २००४ में [[केदार सम्मान]] से सम्मानित किया गया है।{{केदार सम्मान से सम्मानित कृतियाँ}}"),
Start: 0,
End: 411,
},
},
ot: highlight.TermLocations{
&highlight.TermLocation{
Term: "पानी",
Pos: 1,
Start: 2,
End: 14,
},
},
size: 200,
},
{
orig: []byte("交换机"),
fragments: []*highlight.Fragment{
&highlight.Fragment{
Orig: []byte("交换机"),
Start: 0,
End: 9,
},
&highlight.Fragment{
Orig: []byte("交换机"),
Start: 3,
End: 9,
},
},
ot: highlight.TermLocations{
&highlight.TermLocation{
Term: "交换",
Pos: 1,
Start: 0,
End: 6,
},
&highlight.TermLocation{
Term: "换机",
Pos: 2,
Start: 3,
End: 9,
},
},
size: 200,
},
}
for _, test := range tests {
fragmenter := NewFragmenter(test.size)
fragments := fragmenter.Fragment(test.orig, test.ot)
if !reflect.DeepEqual(fragments, test.fragments) {
t.Errorf("expected %#v, got %#v", test.fragments, fragments)
for _, fragment := range fragments {
t.Logf("frag: %s", fragment.Orig[fragment.Start:fragment.End])
t.Logf("frag: %d - %d", fragment.Start, fragment.End)
}
}
}
}
func TestSimpleFragmenterWithSize(t *testing.T) {
tests := []struct {
orig []byte
fragments []*highlight.Fragment
ot highlight.TermLocations
}{
{
orig: []byte("this is a test"),
fragments: []*highlight.Fragment{
&highlight.Fragment{
Orig: []byte("this is a test"),
Start: 0,
End: 5,
},
&highlight.Fragment{
Orig: []byte("this is a test"),
Start: 9,
End: 14,
},
},
ot: highlight.TermLocations{
&highlight.TermLocation{
Term: "this",
Pos: 1,
Start: 0,
End: 5,
},
&highlight.TermLocation{
Term: "test",
Pos: 4,
Start: 10,
End: 14,
},
},
},
}
fragmenter := NewFragmenter(5)
for _, test := range tests {
fragments := fragmenter.Fragment(test.orig, test.ot)
if !reflect.DeepEqual(fragments, test.fragments) {
t.Errorf("expected %#v, got %#v", test.fragments, fragments)
for _, fragment := range fragments {
t.Logf("frag: %#v", fragment)
}
}
}
}