Switch from Godep to go vendoring

vendor/github.com/blevesearch/bleve/index/store/leveldb/batch.go

@@ -0,0 +1,55 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build leveldb full
+
+package leveldb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/jmhodges/levigo"
+)
+
+type Batch struct {
+	w     *Writer
+	merge *store.EmulatedMerge
+	batch *levigo.WriteBatch
+}
+
+func (b *Batch) Set(key, val []byte) {
+	b.batch.Put(key, val)
+}
+
+func (b *Batch) Delete(key []byte) {
+	b.batch.Delete(key)
+}
+
+func (b *Batch) Merge(key, val []byte) {
+	b.merge.Merge(key, val)
+}
+
+func (b *Batch) Execute() error {
+	// first process merges
+	ops, err := b.merge.ExecuteDeferred(b.w)
+	if err != nil {
+		return err
+	}
+	for _, op := range ops {
+		b.batch.Put(op.K, op.V)
+	}
+
+	wopts := defaultWriteOptions()
+	defer wopts.Close()
+	err = b.w.store.db.Write(wopts, b.batch)
+	return err
+}
+
+func (b *Batch) Close() error {
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/leveldb/iterator.go

@@ -0,0 +1,78 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build leveldb full
+
+package leveldb
+
+import (
+	"github.com/jmhodges/levigo"
+)
+
+type Iterator struct {
+	store    *Store
+	iterator *levigo.Iterator
+}
+
+func newIterator(store *Store) *Iterator {
+	ropts := defaultReadOptions()
+	rv := Iterator{
+		store:    store,
+		iterator: store.db.NewIterator(ropts),
+	}
+	ropts.Close()
+	return &rv
+}
+
+func newIteratorWithSnapshot(store *Store, snapshot *levigo.Snapshot) *Iterator {
+	options := defaultReadOptions()
+	options.SetSnapshot(snapshot)
+	rv := Iterator{
+		store:    store,
+		iterator: store.db.NewIterator(options),
+	}
+	options.Close()
+	return &rv
+}
+
+func (ldi *Iterator) SeekFirst() {
+	ldi.iterator.SeekToFirst()
+}
+
+func (ldi *Iterator) Seek(key []byte) {
+	ldi.iterator.Seek(key)
+}
+
+func (ldi *Iterator) Next() {
+	ldi.iterator.Next()
+}
+
+func (ldi *Iterator) Current() ([]byte, []byte, bool) {
+	if ldi.Valid() {
+		return ldi.Key(), ldi.Value(), true
+	}
+	return nil, nil, false
+}
+
+func (ldi *Iterator) Key() []byte {
+	return ldi.iterator.Key()
+}
+
+func (ldi *Iterator) Value() []byte {
+	return ldi.iterator.Value()
+}
+
+func (ldi *Iterator) Valid() bool {
+	return ldi.iterator.Valid()
+}
+
+func (ldi *Iterator) Close() error {
+	ldi.iterator.Close()
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/leveldb/reader.go

@@ -0,0 +1,48 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build leveldb full
+
+package leveldb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/jmhodges/levigo"
+)
+
+type Reader struct {
+	store    *Store
+	snapshot *levigo.Snapshot
+}
+
+func newReader(store *Store) (*Reader, error) {
+	return &Reader{
+		store:    store,
+		snapshot: store.db.NewSnapshot(),
+	}, nil
+}
+
+func (r *Reader) BytesSafeAfterClose() bool {
+	return true
+}
+
+func (r *Reader) Get(key []byte) ([]byte, error) {
+	return r.store.getWithSnapshot(key, r.snapshot)
+}
+
+func (r *Reader) Iterator(key []byte) store.KVIterator {
+	rv := newIteratorWithSnapshot(r.store, r.snapshot)
+	rv.Seek(key)
+	return rv
+}
+
+func (r *Reader) Close() error {
+	r.store.db.ReleaseSnapshot(r.snapshot)
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/leveldb/store.go

@@ -0,0 +1,174 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build leveldb full
+
+package leveldb
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/registry"
+	"github.com/jmhodges/levigo"
+)
+
+const Name = "leveldb"
+
+type Store struct {
+	path   string
+	opts   *levigo.Options
+	db     *levigo.DB
+	writer sync.Mutex
+	mo     store.MergeOperator
+}
+
+func New(path string, config map[string]interface{}) (*Store, error) {
+	rv := Store{
+		path: path,
+		opts: levigo.NewOptions(),
+	}
+
+	_, err := applyConfig(rv.opts, config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &rv, nil
+}
+
+func (ldbs *Store) Open() error {
+	var err error
+	ldbs.db, err = levigo.Open(ldbs.path, ldbs.opts)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ldbs *Store) SetMergeOperator(mo store.MergeOperator) {
+	ldbs.mo = mo
+}
+
+func (ldbs *Store) get(key []byte) ([]byte, error) {
+	options := defaultReadOptions()
+	b, err := ldbs.db.Get(options, key)
+	options.Close()
+	return b, err
+}
+
+func (ldbs *Store) getWithSnapshot(key []byte, snapshot *levigo.Snapshot) ([]byte, error) {
+	options := defaultReadOptions()
+	options.SetSnapshot(snapshot)
+	b, err := ldbs.db.Get(options, key)
+	options.Close()
+	return b, err
+}
+
+func (ldbs *Store) set(key, val []byte) error {
+	ldbs.writer.Lock()
+	defer ldbs.writer.Unlock()
+	return ldbs.setlocked(key, val)
+}
+
+func (ldbs *Store) setlocked(key, val []byte) error {
+	options := defaultWriteOptions()
+	err := ldbs.db.Put(options, key, val)
+	options.Close()
+	return err
+}
+
+func (ldbs *Store) delete(key []byte) error {
+	ldbs.writer.Lock()
+	defer ldbs.writer.Unlock()
+	return ldbs.deletelocked(key)
+}
+
+func (ldbs *Store) deletelocked(key []byte) error {
+	options := defaultWriteOptions()
+	err := ldbs.db.Delete(options, key)
+	options.Close()
+	return err
+}
+
+func (ldbs *Store) Close() error {
+	ldbs.db.Close()
+	ldbs.opts.Close()
+	return nil
+}
+
+func (ldbs *Store) iterator(key []byte) store.KVIterator {
+	rv := newIterator(ldbs)
+	rv.Seek(key)
+	return rv
+}
+
+func (ldbs *Store) Reader() (store.KVReader, error) {
+	return newReader(ldbs)
+}
+
+func (ldbs *Store) Writer() (store.KVWriter, error) {
+	return newWriter(ldbs)
+}
+
+func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify path")
+	}
+	return New(path, config)
+}
+
+func init() {
+	registry.RegisterKVStore(Name, StoreConstructor)
+}
+
+func applyConfig(o *levigo.Options, config map[string]interface{}) (
+	*levigo.Options, error) {
+
+	cim, ok := config["create_if_missing"].(bool)
+	if ok {
+		o.SetCreateIfMissing(cim)
+	}
+
+	eie, ok := config["error_if_exists"].(bool)
+	if ok {
+		o.SetErrorIfExists(eie)
+	}
+
+	wbs, ok := config["write_buffer_size"].(float64)
+	if ok {
+		o.SetWriteBufferSize(int(wbs))
+	}
+
+	bs, ok := config["block_size"].(float64)
+	if ok {
+		o.SetBlockSize(int(bs))
+	}
+
+	bri, ok := config["block_restart_interval"].(float64)
+	if ok {
+		o.SetBlockRestartInterval(int(bri))
+	}
+
+	lcc, ok := config["lru_cache_capacity"].(float64)
+	if ok {
+		lruCache := levigo.NewLRUCache(int(lcc))
+		o.SetCache(lruCache)
+	}
+
+	bfbpk, ok := config["bloom_filter_bits_per_key"].(float64)
+	if ok {
+		bf := levigo.NewBloomFilter(int(bfbpk))
+		o.SetFilterPolicy(bf)
+	}
+
+	return o, nil
+}

vendor/github.com/blevesearch/bleve/index/store/leveldb/store_test.go

@@ -0,0 +1,298 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build leveldb full
+
+package leveldb
+
+import (
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+)
+
+var leveldbTestOptions = map[string]interface{}{
+	"create_if_missing": true,
+}
+
+func TestLevelDBStore(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	s, err := New("test", leveldbTestOptions)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := s.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	CommonTestKVStore(t, s)
+}
+
+func TestReaderIsolation(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	s, err := New("test", leveldbTestOptions)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := s.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	CommonTestReaderIsolation(t, s)
+}
+
+func CommonTestKVStore(t *testing.T, s store.KVStore) {
+
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Set([]byte("z"), []byte("val-z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Delete([]byte("z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	batch := writer.NewBatch()
+	batch.Set([]byte("b"), []byte("val-b"))
+	batch.Set([]byte("c"), []byte("val-c"))
+	batch.Set([]byte("d"), []byte("val-d"))
+	batch.Set([]byte("e"), []byte("val-e"))
+	batch.Set([]byte("f"), []byte("val-f"))
+	batch.Set([]byte("g"), []byte("val-g"))
+	batch.Set([]byte("h"), []byte("val-h"))
+	batch.Set([]byte("i"), []byte("val-i"))
+	batch.Set([]byte("j"), []byte("val-j"))
+
+	err = batch.Execute()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	it := reader.Iterator([]byte("b"))
+	key, val, valid := it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "b" {
+		t.Fatalf("expected key b, got %s", key)
+	}
+	if string(val) != "val-b" {
+		t.Fatalf("expected value val-b, got %s", val)
+	}
+
+	it.Next()
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "c" {
+		t.Fatalf("expected key c, got %s", key)
+	}
+	if string(val) != "val-c" {
+		t.Fatalf("expected value val-c, got %s", val)
+	}
+
+	it.Seek([]byte("i"))
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "i" {
+		t.Fatalf("expected key i, got %s", key)
+	}
+	if string(val) != "val-i" {
+		t.Fatalf("expected value val-i, got %s", val)
+	}
+
+	err = it.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
+	// insert a kv pair
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create an isolated reader
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	// verify that we see the value already inserted
+	val, err := reader.Get([]byte("a"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-a")) {
+		t.Errorf("expected val-a, got nil")
+	}
+
+	// verify that an iterator sees it
+	count := 0
+	it := reader.Iterator([]byte{0})
+	defer func() {
+		err := it.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it.Valid() {
+		it.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+	// add something after the reader was created
+	writer, err = s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("b"), []byte("val-b"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ensure that a newer reader sees it
+	newReader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := newReader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	val, err = newReader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-b")) {
+		t.Errorf("expected val-b, got nil")
+	}
+
+	// ensure that the director iterator sees it
+	count = 0
+	it2 := newReader.Iterator([]byte{0})
+	defer func() {
+		err := it2.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it2.Valid() {
+		it2.Next()
+		count++
+	}
+	if count != 2 {
+		t.Errorf("expected iterator to see 2, saw %d", count)
+	}
+
+	// but that the isolated reader does not
+	val, err = reader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if val != nil {
+		t.Errorf("expected nil, got %v", val)
+	}
+
+	// and ensure that the iterator on the isolated reader also does not
+	count = 0
+	it3 := reader.Iterator([]byte{0})
+	defer func() {
+		err := it3.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it3.Valid() {
+		it3.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+}

vendor/github.com/blevesearch/bleve/index/store/leveldb/util.go

@@ -0,0 +1,28 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build leveldb full
+
+package leveldb
+
+import (
+	"github.com/jmhodges/levigo"
+)
+
+func defaultWriteOptions() *levigo.WriteOptions {
+	wo := levigo.NewWriteOptions()
+	// request fsync on write for safety
+	wo.SetSync(true)
+	return wo
+}
+
+func defaultReadOptions() *levigo.ReadOptions {
+	ro := levigo.NewReadOptions()
+	return ro
+}

vendor/github.com/blevesearch/bleve/index/store/leveldb/writer.go

@@ -0,0 +1,65 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build leveldb full
+
+package leveldb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/jmhodges/levigo"
+)
+
+type Writer struct {
+	store *Store
+}
+
+func newWriter(store *Store) (*Writer, error) {
+	store.writer.Lock()
+	return &Writer{
+		store: store,
+	}, nil
+}
+
+func (w *Writer) BytesSafeAfterClose() bool {
+	return true
+}
+
+func (w *Writer) Set(key, val []byte) error {
+	return w.store.setlocked(key, val)
+}
+
+func (w *Writer) Delete(key []byte) error {
+	return w.store.deletelocked(key)
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	rv := Batch{
+		w:     w,
+		merge: store.NewEmulatedMerge(w.store.mo),
+		batch: levigo.NewWriteBatch(),
+	}
+	return &rv
+}
+
+func (w *Writer) Close() error {
+	w.store.writer.Unlock()
+	return nil
+}
+
+// these two methods can safely read using the regular
+// methods without a read transaction, because we know
+// that no one else is writing but us
+func (w *Writer) Get(key []byte) ([]byte, error) {
+	return w.store.get(key)
+}
+
+func (w *Writer) Iterator(key []byte) store.KVIterator {
+	return w.store.iterator(key)
+}