Switch from Godep to go vendoring

vendor/github.com/blevesearch/bleve/analysis/language/ja/analyzer_ja.go

@@ -0,0 +1,39 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build kagome full
+
+package ja
+
+import (
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/analysis/token_filters/unicode_normalize"
+	"github.com/blevesearch/bleve/registry"
+)
+
+const AnalyzerName = "ja"
+
+func AnalyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) {
+	kagomeTokenizer, err := cache.TokenizerNamed(TokenizerName)
+	if err != nil {
+		return nil, err
+	}
+	normalizeFilter := unicode_normalize.MustNewUnicodeNormalizeFilter(unicode_normalize.NFKD)
+	rv := analysis.Analyzer{
+		Tokenizer: kagomeTokenizer,
+		TokenFilters: []analysis.TokenFilter{
+			normalizeFilter,
+		},
+	}
+	return &rv, nil
+}
+
+func init() {
+	registry.RegisterAnalyzer(AnalyzerName, AnalyzerConstructor)
+}

vendor/github.com/blevesearch/bleve/analysis/language/ja/analyzer_ja_test.go

@@ -0,0 +1,71 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build kagome full
+
+package ja
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+)
+
+func TestJaAnalyzer(t *testing.T) {
+	tests := []struct {
+		input  []byte
+		output analysis.TokenStream
+	}{
+		{
+			input: []byte("こんにちは世界"),
+			output: analysis.TokenStream{
+				&analysis.Token{
+					Term:     []byte("こんにちは"),
+					Type:     analysis.Ideographic,
+					Position: 1,
+					Start:    0,
+					End:      15,
+				},
+				&analysis.Token{
+					Term:     []byte("世界"),
+					Type:     analysis.Ideographic,
+					Position: 2,
+					Start:    15,
+					End:      21,
+				},
+			},
+		},
+		{
+			input: []byte("ｶﾀｶﾅ"),
+			output: analysis.TokenStream{
+				&analysis.Token{
+					Term:     []byte("カタカナ"),
+					Type:     analysis.Ideographic,
+					Position: 1,
+					Start:    0,
+					End:      12,
+				},
+			},
+		},
+	}
+
+	cache := registry.NewCache()
+	for _, test := range tests {
+		analyzer, err := cache.AnalyzerNamed(AnalyzerName)
+		if err != nil {
+			t.Fatal(err)
+		}
+		actual := analyzer.Analyze(test.input)
+		if !reflect.DeepEqual(actual, test.output) {
+			t.Errorf("expected %v, got %v", test.output, actual)
+		}
+	}
+}

vendor/github.com/blevesearch/bleve/analysis/language/ja/ja_morph_kagome.go

@@ -0,0 +1,81 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build kagome full
+
+package ja
+
+import (
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/registry"
+
+	"github.com/ikawaha/kagome"
+)
+
+const TokenizerName = "kagome"
+
+type KagomeMorphTokenizer struct {
+	tok *kagome.Tokenizer
+}
+
+func NewKagomeMorphTokenizer() *KagomeMorphTokenizer {
+	return &KagomeMorphTokenizer{
+		tok: kagome.NewTokenizer(),
+	}
+}
+
+func NewKagomeMorphTokenizerWithUserDic(userdic *kagome.UserDic) *KagomeMorphTokenizer {
+	k := kagome.NewTokenizer()
+	k.SetUserDic(userdic)
+	return &KagomeMorphTokenizer{
+		tok: k,
+	}
+}
+
+func (t *KagomeMorphTokenizer) Tokenize(input []byte) analysis.TokenStream {
+	var (
+		morphs    []kagome.Token
+		prevstart int
+	)
+
+	rv := make(analysis.TokenStream, 0, len(input))
+	if len(input) < 1 {
+		return rv
+	}
+
+	morphs = t.tok.Tokenize(string(input))
+
+	for i, m := range morphs {
+		if m.Surface == "EOS" || m.Surface == "BOS" {
+			continue
+		}
+
+		surfacelen := len(m.Surface)
+		token := &analysis.Token{
+			Term:     []byte(m.Surface),
+			Position: i,
+			Start:    prevstart,
+			End:      prevstart + surfacelen,
+			Type:     analysis.Ideographic,
+		}
+
+		prevstart = prevstart + surfacelen
+		rv = append(rv, token)
+	}
+
+	return rv
+}
+
+func KagomeMorphTokenizerConstructor(config map[string]interface{}, cache *registry.Cache) (analysis.Tokenizer, error) {
+	return NewKagomeMorphTokenizer(), nil
+}
+
+func init() {
+	registry.RegisterTokenizer(TokenizerName, KagomeMorphTokenizerConstructor)
+}

vendor/github.com/blevesearch/bleve/analysis/language/ja/ja_morph_kagome_test.go

@@ -0,0 +1,56 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build kagome full
+
+package ja
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/analysis"
+)
+
+func TestKagome(t *testing.T) {
+
+	tests := []struct {
+		input  []byte
+		output analysis.TokenStream
+	}{
+		{
+			[]byte("こんにちは世界"),
+			analysis.TokenStream{
+				{
+					Start:    0,
+					End:      15,
+					Term:     []byte("こんにちは"),
+					Position: 1,
+					Type:     analysis.Ideographic,
+				},
+				{
+					Start:    15,
+					End:      21,
+					Term:     []byte("世界"),
+					Position: 2,
+					Type:     analysis.Ideographic,
+				},
+			},
+		},
+	}
+
+	tokenizer := NewKagomeMorphTokenizer()
+	for _, test := range tests {
+		actuals := tokenizer.Tokenize(test.input)
+
+		if !reflect.DeepEqual(actuals, test.output) {
+			t.Errorf("Expected %v, got %v for %s", test.output, actuals, string(test.input))
+		}
+	}
+}