Switch from Godep to go vendoring

vendor/github.com/blevesearch/bleve/index/index.go

@@ -0,0 +1,146 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package index
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/blevesearch/bleve/document"
+)
+
+type Index interface {
+	Open() error
+	Close() error
+
+	DocCount() (uint64, error)
+
+	Update(doc *document.Document) error
+	Delete(id string) error
+	Batch(batch *Batch) error
+
+	SetInternal(key, val []byte) error
+	DeleteInternal(key []byte) error
+
+	DumpAll() chan interface{}
+	DumpDoc(id string) chan interface{}
+	DumpFields() chan interface{}
+
+	Reader() (IndexReader, error)
+
+	Stats() json.Marshaler
+}
+
+type IndexReader interface {
+	TermFieldReader(term []byte, field string) (TermFieldReader, error)
+	DocIDReader(start, end string) (DocIDReader, error)
+
+	FieldDict(field string) (FieldDict, error)
+	FieldDictRange(field string, startTerm []byte, endTerm []byte) (FieldDict, error)
+	FieldDictPrefix(field string, termPrefix []byte) (FieldDict, error)
+
+	Document(id string) (*document.Document, error)
+	DocumentFieldTerms(id string) (FieldTerms, error)
+
+	Fields() ([]string, error)
+
+	GetInternal(key []byte) ([]byte, error)
+
+	DocCount() uint64
+
+	Close() error
+}
+
+type FieldTerms map[string][]string
+
+type TermFieldVector struct {
+	Field string
+	Pos   uint64
+	Start uint64
+	End   uint64
+}
+
+type TermFieldDoc struct {
+	Term    string
+	ID      string
+	Freq    uint64
+	Norm    float64
+	Vectors []*TermFieldVector
+}
+
+type TermFieldReader interface {
+	Next() (*TermFieldDoc, error)
+	Advance(ID string) (*TermFieldDoc, error)
+	Count() uint64
+	Close() error
+}
+
+type DictEntry struct {
+	Term  string
+	Count uint64
+}
+
+type FieldDict interface {
+	Next() (*DictEntry, error)
+	Close() error
+}
+
+type DocIDReader interface {
+	Next() (string, error)
+	Advance(ID string) (string, error)
+	Close() error
+}
+
+type Batch struct {
+	IndexOps    map[string]*document.Document
+	InternalOps map[string][]byte
+}
+
+func NewBatch() *Batch {
+	return &Batch{
+		IndexOps:    make(map[string]*document.Document),
+		InternalOps: make(map[string][]byte),
+	}
+}
+
+func (b *Batch) Update(doc *document.Document) {
+	b.IndexOps[doc.ID] = doc
+}
+
+func (b *Batch) Delete(id string) {
+	b.IndexOps[id] = nil
+}
+
+func (b *Batch) SetInternal(key, val []byte) {
+	b.InternalOps[string(key)] = val
+}
+
+func (b *Batch) DeleteInternal(key []byte) {
+	b.InternalOps[string(key)] = nil
+}
+
+func (b *Batch) String() string {
+	rv := fmt.Sprintf("Batch (%d ops, %d internal ops)\n", len(b.IndexOps), len(b.InternalOps))
+	for k, v := range b.IndexOps {
+		if v != nil {
+			rv += fmt.Sprintf("\tINDEX - '%s'\n", k)
+		} else {
+			rv += fmt.Sprintf("\tDELETE - '%s'\n", k)
+		}
+	}
+	for k, v := range b.InternalOps {
+		if v != nil {
+			rv += fmt.Sprintf("\tSET INTERNAL - '%s'\n", k)
+		} else {
+			rv += fmt.Sprintf("\tDELETE INTERNAL - '%s'\n", k)
+		}
+	}
+	return rv
+}

vendor/github.com/blevesearch/bleve/index/store/batch.go

@@ -0,0 +1,69 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package store
+
+type op struct {
+	K []byte
+	V []byte
+}
+
+type EmulatedBatch struct {
+	w     KVWriter
+	ops   []*op
+	merge *EmulatedMerge
+}
+
+func NewEmulatedBatch(w KVWriter, mo MergeOperator) *EmulatedBatch {
+	return &EmulatedBatch{
+		w:     w,
+		ops:   make([]*op, 0, 1000),
+		merge: NewEmulatedMerge(mo),
+	}
+}
+
+func (b *EmulatedBatch) Set(key, val []byte) {
+	b.ops = append(b.ops, &op{key, val})
+}
+
+func (b *EmulatedBatch) Delete(key []byte) {
+	b.ops = append(b.ops, &op{key, nil})
+}
+
+func (b *EmulatedBatch) Merge(key, val []byte) {
+	b.merge.Merge(key, val)
+}
+
+func (b *EmulatedBatch) Execute() error {
+	// first process merges
+	err := b.merge.Execute(b.w)
+	if err != nil {
+		return err
+	}
+
+	// now apply all the ops
+	for _, op := range b.ops {
+		if op.V != nil {
+			err := b.w.Set(op.K, op.V)
+			if err != nil {
+				return err
+			}
+		} else {
+			err := b.w.Delete(op.K)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (b *EmulatedBatch) Close() error {
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/boltdb/iterator.go

@@ -0,0 +1,58 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package boltdb
+
+import (
+	"github.com/boltdb/bolt"
+)
+
+type Iterator struct {
+	store  *Store
+	tx     *bolt.Tx
+	cursor *bolt.Cursor
+	valid  bool
+	key    []byte
+	val    []byte
+}
+
+func (i *Iterator) SeekFirst() {
+	i.key, i.val = i.cursor.First()
+	i.valid = (i.key != nil)
+}
+
+func (i *Iterator) Seek(k []byte) {
+	i.key, i.val = i.cursor.Seek(k)
+	i.valid = (i.key != nil)
+}
+
+func (i *Iterator) Next() {
+	i.key, i.val = i.cursor.Next()
+	i.valid = (i.key != nil)
+}
+
+func (i *Iterator) Current() ([]byte, []byte, bool) {
+	return i.key, i.val, i.valid
+}
+
+func (i *Iterator) Key() []byte {
+	return i.key
+}
+
+func (i *Iterator) Value() []byte {
+	return i.val
+}
+
+func (i *Iterator) Valid() bool {
+	return i.valid
+}
+
+func (i *Iterator) Close() error {
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/boltdb/reader.go

@@ -0,0 +1,47 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package boltdb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/boltdb/bolt"
+)
+
+type Reader struct {
+	store *Store
+	tx    *bolt.Tx
+}
+
+func (r *Reader) BytesSafeAfterClose() bool {
+	return false
+}
+
+func (r *Reader) Get(key []byte) ([]byte, error) {
+	rv := r.tx.Bucket([]byte(r.store.bucket)).Get(key)
+	return rv, nil
+}
+
+func (r *Reader) Iterator(key []byte) store.KVIterator {
+	b := r.tx.Bucket([]byte(r.store.bucket))
+	cursor := b.Cursor()
+
+	rv := &Iterator{
+		store:  r.store,
+		tx:     r.tx,
+		cursor: cursor,
+	}
+
+	rv.Seek(key)
+	return rv
+}
+
+func (r *Reader) Close() error {
+	return r.tx.Rollback()
+}

vendor/github.com/blevesearch/bleve/index/store/boltdb/store.go

@@ -0,0 +1,112 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package boltdb
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/registry"
+	"github.com/boltdb/bolt"
+)
+
+const Name = "boltdb"
+
+type Store struct {
+	path   string
+	bucket string
+	db     *bolt.DB
+	writer sync.Mutex
+	mo     store.MergeOperator
+}
+
+func New(path string, bucket string) *Store {
+	rv := Store{
+		path:   path,
+		bucket: bucket,
+	}
+	return &rv
+}
+
+func (bs *Store) Open() error {
+
+	var err error
+	bs.db, err = bolt.Open(bs.path, 0600, nil)
+	if err != nil {
+		return err
+	}
+
+	err = bs.db.Update(func(tx *bolt.Tx) error {
+		_, err := tx.CreateBucketIfNotExists([]byte(bs.bucket))
+
+		return err
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (bs *Store) SetMergeOperator(mo store.MergeOperator) {
+	bs.mo = mo
+}
+
+func (bs *Store) Close() error {
+	return bs.db.Close()
+}
+
+func (bs *Store) Reader() (store.KVReader, error) {
+	tx, err := bs.db.Begin(false)
+	if err != nil {
+		return nil, err
+	}
+	return &Reader{
+		store: bs,
+		tx:    tx,
+	}, nil
+}
+
+func (bs *Store) Writer() (store.KVWriter, error) {
+	bs.writer.Lock()
+	tx, err := bs.db.Begin(true)
+	if err != nil {
+		bs.writer.Unlock()
+		return nil, err
+	}
+	reader := &Reader{
+		store: bs,
+		tx:    tx,
+	}
+	return &Writer{
+		store:  bs,
+		tx:     tx,
+		reader: reader,
+	}, nil
+}
+
+func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify path")
+	}
+
+	bucket, ok := config["bucket"].(string)
+	if !ok {
+		bucket = "bleve"
+	}
+
+	return New(path, bucket), nil
+}
+
+func init() {
+	registry.RegisterKVStore(Name, StoreConstructor)
+}

vendor/github.com/blevesearch/bleve/index/store/boltdb/store_test.go

@@ -0,0 +1,272 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package boltdb
+
+import (
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+)
+
+func TestStore(t *testing.T) {
+	s := New("test", "bleve")
+	err := s.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := os.RemoveAll("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	CommonTestKVStore(t, s)
+}
+
+func TestReaderIsolation(t *testing.T) {
+	s := New("test", "bleve")
+	err := s.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := os.RemoveAll("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	CommonTestReaderIsolation(t, s)
+}
+
+func CommonTestKVStore(t *testing.T, s store.KVStore) {
+
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Set([]byte("z"), []byte("val-z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Delete([]byte("z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	batch := writer.NewBatch()
+	batch.Set([]byte("b"), []byte("val-b"))
+	batch.Set([]byte("c"), []byte("val-c"))
+	batch.Set([]byte("d"), []byte("val-d"))
+	batch.Set([]byte("e"), []byte("val-e"))
+	batch.Set([]byte("f"), []byte("val-f"))
+	batch.Set([]byte("g"), []byte("val-g"))
+	batch.Set([]byte("h"), []byte("val-h"))
+	batch.Set([]byte("i"), []byte("val-i"))
+	batch.Set([]byte("j"), []byte("val-j"))
+
+	err = batch.Execute()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	it := reader.Iterator([]byte("b"))
+	key, val, valid := it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "b" {
+		t.Fatalf("expected key b, got %s", key)
+	}
+	if string(val) != "val-b" {
+		t.Fatalf("expected value val-b, got %s", val)
+	}
+
+	it.Next()
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "c" {
+		t.Fatalf("expected key c, got %s", key)
+	}
+	if string(val) != "val-c" {
+		t.Fatalf("expected value val-c, got %s", val)
+	}
+
+	it.Seek([]byte("i"))
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "i" {
+		t.Fatalf("expected key i, got %s", key)
+	}
+	if string(val) != "val-i" {
+		t.Fatalf("expected value val-i, got %s", val)
+	}
+
+	err = it.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
+	// insert a kv pair
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create an isolated reader
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	// verify that we see the value already inserted
+	val, err := reader.Get([]byte("a"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-a")) {
+		t.Errorf("expected val-a, got nil")
+	}
+
+	// verify that an iterator sees it
+	count := 0
+	it := reader.Iterator([]byte{0})
+	defer func() {
+		err := it.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it.Valid() {
+		it.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+	// add something after the reader was created
+	writer, err = s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("b"), []byte("val-b"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ensure that a newer reader sees it
+	newReader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := newReader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	val, err = newReader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-b")) {
+		t.Errorf("expected val-b, got nil")
+	}
+
+	// ensure that the director iterator sees it
+	count = 0
+	it2 := newReader.Iterator([]byte{0})
+	defer func() {
+		err := it2.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it2.Valid() {
+		it2.Next()
+		count++
+	}
+	if count != 2 {
+		t.Errorf("expected iterator to see 2, saw %d", count)
+	}
+
+	// but that the isolated reader does not
+	val, err = reader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if val != nil {
+		t.Errorf("expected nil, got %v", val)
+	}
+
+	// and ensure that the iterator on the isolated reader also does not
+	count = 0
+	it3 := reader.Iterator([]byte{0})
+	defer func() {
+		err := it3.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it3.Valid() {
+		it3.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+}

vendor/github.com/blevesearch/bleve/index/store/boltdb/writer.go

@@ -0,0 +1,50 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package boltdb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/boltdb/bolt"
+)
+
+type Writer struct {
+	store  *Store
+	tx     *bolt.Tx
+	reader *Reader
+}
+
+func (w *Writer) Set(key, val []byte) error {
+	return w.tx.Bucket([]byte(w.store.bucket)).Put(key, val)
+}
+
+func (w *Writer) Delete(key []byte) error {
+	return w.tx.Bucket([]byte(w.store.bucket)).Delete(key)
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	return store.NewEmulatedBatch(w, w.store.mo)
+}
+
+func (w *Writer) Close() error {
+	w.store.writer.Unlock()
+	return w.tx.Commit()
+}
+
+func (w *Writer) BytesSafeAfterClose() bool {
+	return w.reader.BytesSafeAfterClose()
+}
+
+func (w *Writer) Get(key []byte) ([]byte, error) {
+	return w.reader.Get(key)
+}
+
+func (w *Writer) Iterator(key []byte) store.KVIterator {
+	return w.reader.Iterator(key)
+}

vendor/github.com/blevesearch/bleve/index/store/cznicb/batch.go

@@ -0,0 +1,88 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the
+//  License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing,
+//  software distributed under the License is distributed on an "AS
+//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+//  express or implied. See the License for the specific language
+//  governing permissions and limitations under the License.
+
+// +build go1.4
+
+package cznicb
+
+import ()
+
+type op struct {
+	k []byte
+	v []byte
+}
+
+type Batch struct {
+	s      *Store
+	ops    []op
+	merges map[string][][]byte
+}
+
+func (b *Batch) Set(k, v []byte) {
+	b.ops = append(b.ops, op{k, v})
+}
+
+func (b *Batch) Delete(k []byte) {
+	b.ops = append(b.ops, op{k, nil})
+}
+
+func (b *Batch) Merge(key, val []byte) {
+	ops, ok := b.merges[string(key)]
+	if ok && len(ops) > 0 {
+		last := ops[len(ops)-1]
+		mergedVal, partialMergeOk := b.s.mo.PartialMerge(key, last, val)
+		if partialMergeOk {
+			// replace last entry with the result of the merge
+			ops[len(ops)-1] = mergedVal
+		} else {
+			// could not partial merge, append this to the end
+			ops = append(ops, val)
+		}
+	} else {
+		ops = [][]byte{val}
+	}
+	b.merges[string(key)] = ops
+}
+
+func (b *Batch) Execute() (err error) {
+	b.s.m.Lock()
+	defer b.s.m.Unlock()
+
+	t := b.s.t
+	for key, mergeOps := range b.merges {
+		k := []byte(key)
+		t.Put(k, func(oldV interface{}, exists bool) (newV interface{}, write bool) {
+			ob := []byte(nil)
+			if exists && oldV != nil {
+				ob = oldV.([]byte)
+			}
+			mergedVal, fullMergeOk := b.s.mo.FullMerge(k, ob, mergeOps)
+			if !fullMergeOk {
+				return nil, false
+			}
+			return mergedVal, true
+		})
+	}
+
+	for _, op := range b.ops {
+		if op.v != nil {
+			t.Set(op.k, op.v)
+		} else {
+			t.Delete(op.k)
+		}
+	}
+
+	return nil
+}
+
+func (b *Batch) Close() error {
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/cznicb/cznicb.go

@@ -0,0 +1,112 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the
+//  License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing,
+//  software distributed under the License is distributed on an "AS
+//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+//  express or implied. See the License for the specific language
+//  governing permissions and limitations under the License.
+
+// +build go1.4
+
+// Package cznicb provides an in-memory implementation of the KVStore
+// interfaces using the cznic/b in-memory btree.  Of note: this
+// implementation does not have reader isolation.
+package cznicb
+
+import (
+	"bytes"
+	"fmt"
+	"sync"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/registry"
+
+	"github.com/cznic/b"
+)
+
+const Name = "cznicb"
+
+const MAX_CONCURRENT_WRITERS = 1
+
+func init() {
+	registry.RegisterKVStore(Name, StoreConstructor)
+}
+
+func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
+	s := &Store{
+		t:                b.TreeNew(itemCompare),
+		availableWriters: make(chan bool, MAX_CONCURRENT_WRITERS),
+	}
+	for i := 0; i < MAX_CONCURRENT_WRITERS; i++ {
+		s.availableWriters <- true
+	}
+	return s, nil
+}
+
+func itemCompare(a, b interface{}) int {
+	return bytes.Compare(a.([]byte), b.([]byte))
+}
+
+type Store struct {
+	availableWriters chan bool
+	m                sync.RWMutex
+	t                *b.Tree
+	mo               store.MergeOperator
+}
+
+func (s *Store) Open() error {
+	return nil
+}
+
+func (s *Store) SetMergeOperator(mo store.MergeOperator) {
+	s.mo = mo
+}
+
+func (s *Store) Reader() (store.KVReader, error) {
+	return &Reader{s: s}, nil
+}
+
+func (s *Store) Writer() (store.KVWriter, error) {
+	available, ok := <-s.availableWriters
+	if !ok || !available {
+		return nil, fmt.Errorf("no available writers")
+	}
+	return &Writer{s: s, r: &Reader{s: s}}, nil
+}
+
+func (s *Store) Close() error {
+	return nil
+}
+
+func (s *Store) get(k []byte) ([]byte, error) {
+	s.m.RLock()
+	defer s.m.RUnlock()
+	v, ok := s.t.Get(k)
+	if !ok || v == nil {
+		return nil, nil
+	}
+	return v.([]byte), nil
+}
+
+func (s *Store) iterator(k []byte) store.KVIterator {
+	iter := &Iterator{s: s}
+	iter.Seek(k)
+	return iter
+}
+
+func (s *Store) set(k, v []byte) (err error) {
+	s.m.Lock()
+	defer s.m.Unlock()
+	s.t.Set(k, v)
+	return nil
+}
+
+func (s *Store) delete(k []byte) (err error) {
+	s.m.Lock()
+	defer s.m.Unlock()
+	s.t.Delete(k)
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/cznicb/cznicb_test.go

@@ -0,0 +1,133 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the
+//  License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing,
+//  software distributed under the License is distributed on an "AS
+//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+//  express or implied. See the License for the specific language
+//  governing permissions and limitations under the License.
+
+package cznicb
+
+import (
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+)
+
+func TestCznicBStore(t *testing.T) {
+	s, err := StoreConstructor(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	CommonTestKVStore(t, s)
+}
+
+func CommonTestKVStore(t *testing.T, s store.KVStore) {
+
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	v, err := writer.Get([]byte("a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(v) != "val-a" {
+		t.Errorf("expected val-a")
+	}
+	v, err = writer.Get([]byte("not-there"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if v != nil {
+		t.Errorf("expected nil v")
+	}
+	err = writer.Set([]byte("z"), []byte("val-z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Delete([]byte("z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	batch := writer.NewBatch()
+	batch.Set([]byte("b"), []byte("val-b"))
+	batch.Set([]byte("c"), []byte("val-c"))
+	batch.Set([]byte("d"), []byte("val-d"))
+	batch.Set([]byte("e"), []byte("val-e"))
+	batch.Set([]byte("f"), []byte("val-f"))
+	batch.Set([]byte("g"), []byte("val-g"))
+	batch.Set([]byte("h"), []byte("val-h"))
+	batch.Set([]byte("i"), []byte("val-i"))
+	batch.Set([]byte("j"), []byte("val-j"))
+
+	err = batch.Execute()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	it := reader.Iterator([]byte("b"))
+	key, val, valid := it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "b" {
+		t.Fatalf("expected key b, got %s", key)
+	}
+	if string(val) != "val-b" {
+		t.Fatalf("expected value val-b, got %s", val)
+	}
+
+	it.Next()
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "c" {
+		t.Fatalf("expected key c, got %s", key)
+	}
+	if string(val) != "val-c" {
+		t.Fatalf("expected value val-c, got %s", val)
+	}
+
+	it.Seek([]byte("i"))
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "i" {
+		t.Fatalf("expected key i, got %s", key)
+	}
+	if string(val) != "val-i" {
+		t.Fatalf("expected value val-i, got %s", val)
+	}
+
+	err = it.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

vendor/github.com/blevesearch/bleve/index/store/cznicb/iterator.go

@@ -0,0 +1,113 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the
+//  License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing,
+//  software distributed under the License is distributed on an "AS
+//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+//  express or implied. See the License for the specific language
+//  governing permissions and limitations under the License.
+
+// +build go1.4
+
+package cznicb
+
+import (
+	"errors"
+
+	"github.com/cznic/b"
+)
+
+var iteratorDoneErr = errors.New("iteratorDoneErr") // A sentinel value.
+
+type Iterator struct { // Assuming that iterators are used single-threaded.
+	s *Store
+	e *b.Enumerator
+
+	currK   interface{}
+	currV   interface{}
+	currErr error
+}
+
+func (i *Iterator) SeekFirst() {
+	i.currK = nil
+	i.currV = nil
+	i.currErr = nil
+
+	var err error
+	i.s.m.RLock()
+	i.e, err = i.s.t.SeekFirst()
+	i.s.m.RUnlock() // cannot defer, must unlock before Next
+	if err != nil {
+		i.currK = nil
+		i.currV = nil
+		i.currErr = iteratorDoneErr
+	}
+
+	i.Next()
+}
+
+func (i *Iterator) Seek(k []byte) {
+	i.currK = nil
+	i.currV = nil
+	i.currErr = nil
+
+	i.s.m.RLock()
+	i.e, _ = i.s.t.Seek(k)
+	i.s.m.RUnlock() // cannot defer, must unlock before Next
+
+	i.Next()
+}
+
+func (i *Iterator) Next() {
+	if i.currErr != nil {
+		i.currK = nil
+		i.currV = nil
+		i.currErr = iteratorDoneErr
+		return
+	}
+
+	i.s.m.RLock()
+	defer i.s.m.RUnlock()
+	i.currK, i.currV, i.currErr = i.e.Next()
+}
+
+func (i *Iterator) Current() ([]byte, []byte, bool) {
+	if i.currErr == iteratorDoneErr ||
+		i.currK == nil ||
+		i.currV == nil {
+		return nil, nil, false
+	}
+
+	return i.currK.([]byte), i.currV.([]byte), true
+}
+
+func (i *Iterator) Key() []byte {
+	k, _, ok := i.Current()
+	if !ok {
+		return nil
+	}
+	return k
+}
+
+func (i *Iterator) Value() []byte {
+	_, v, ok := i.Current()
+	if !ok {
+		return nil
+	}
+	return v
+}
+
+func (i *Iterator) Valid() bool {
+	_, _, ok := i.Current()
+	return ok
+}
+
+func (i *Iterator) Close() error {
+	if i.e != nil {
+		i.e.Close()
+	}
+	i.e = nil
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/cznicb/reader.go

@@ -0,0 +1,44 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the
+//  License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing,
+//  software distributed under the License is distributed on an "AS
+//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+//  express or implied. See the License for the specific language
+//  governing permissions and limitations under the License.
+
+// +build go1.4
+
+package cznicb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+)
+
+type Reader struct {
+	s *Store
+}
+
+func newReader(s *Store) (*Reader, error) {
+	return &Reader{
+		s: s,
+	}, nil
+}
+
+func (r *Reader) BytesSafeAfterClose() bool {
+	return false
+}
+
+func (r *Reader) Get(key []byte) ([]byte, error) {
+	return r.s.get(key)
+}
+
+func (r *Reader) Iterator(key []byte) store.KVIterator {
+	return r.s.iterator(key)
+}
+
+func (r *Reader) Close() error {
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/cznicb/writer.go

@@ -0,0 +1,55 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build go1.4
+
+package cznicb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+)
+
+type Writer struct {
+	s *Store
+	r *Reader
+}
+
+func (w *Writer) BytesSafeAfterClose() bool {
+	return false
+}
+
+func (w *Writer) Set(key, val []byte) error {
+	return w.s.set(key, val)
+}
+
+func (w *Writer) Delete(key []byte) error {
+	return w.s.delete(key)
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	return &Batch{
+		s:      w.s,
+		ops:    make([]op, 0, 1000),
+		merges: make(map[string][][]byte),
+	}
+}
+
+func (w *Writer) Close() error {
+	w.s.availableWriters <- true
+	w.s = nil
+	return nil
+}
+
+func (w *Writer) Get(key []byte) ([]byte, error) {
+	return w.r.s.get(key)
+}
+
+func (w *Writer) Iterator(key []byte) store.KVIterator {
+	return w.r.s.iterator(key)
+}

vendor/github.com/blevesearch/bleve/index/store/forestdb/batch.go

@@ -0,0 +1,86 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build forestdb
+
+package forestdb
+
+import (
+	"fmt"
+)
+
+type op struct {
+	k []byte
+	v []byte
+}
+
+type Batch struct {
+	s      *Store
+	ops    []op
+	merges map[string][][]byte
+}
+
+func (b *Batch) Set(k, v []byte) {
+	b.ops = append(b.ops, op{k, v})
+}
+
+func (b *Batch) Delete(k []byte) {
+	b.ops = append(b.ops, op{k, nil})
+}
+
+func (b *Batch) Merge(key, val []byte) {
+	ops, ok := b.merges[string(key)]
+	if ok && len(ops) > 0 {
+		last := ops[len(ops)-1]
+		mergedVal, partialMergeOk := b.s.mo.PartialMerge(key, last, val)
+		if partialMergeOk {
+			// replace last entry with the result of the merge
+			ops[len(ops)-1] = mergedVal
+		} else {
+			// could not partial merge, append this to the end
+			ops = append(ops, val)
+		}
+	} else {
+		ops = [][]byte{val}
+	}
+	b.merges[string(key)] = ops
+}
+
+func (b *Batch) Execute() (err error) {
+
+	for k, mergeOps := range b.merges {
+		kb := []byte(k)
+		existingVal, err := b.s.get(kb)
+		if err != nil {
+			return err
+		}
+		mergedVal, fullMergeOk := b.s.mo.FullMerge(kb, existingVal, mergeOps)
+		if !fullMergeOk {
+			return fmt.Errorf("merge operator returned failure")
+		}
+		err = b.s.setlocked(kb, mergedVal)
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, op := range b.ops {
+		if op.v != nil {
+			b.s.setlocked(op.k, op.v)
+		} else {
+			b.s.deletelocked(op.k)
+		}
+	}
+
+	return b.s.commit()
+}
+
+func (b *Batch) Close() error {
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/forestdb/iterator.go

@@ -0,0 +1,117 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build forestdb
+
+package forestdb
+
+import (
+	"github.com/couchbase/goforestdb"
+)
+
+type Iterator struct {
+	store    *Store
+	snapshot *forestdb.KVStore
+	iterator *forestdb.Iterator
+	curr     *forestdb.Doc
+	valid    bool
+}
+
+func newIterator(store *Store) *Iterator {
+	itr, err := store.dbkv.IteratorInit([]byte{}, nil, forestdb.ITR_NONE)
+	rv := Iterator{
+		store:    store,
+		iterator: itr,
+		valid:    err == nil,
+	}
+	return &rv
+}
+
+func newIteratorWithSnapshot(store *Store, snapshot *forestdb.KVStore) *Iterator {
+	itr, err := snapshot.IteratorInit([]byte{}, nil, forestdb.ITR_NONE)
+	rv := Iterator{
+		store:    store,
+		iterator: itr,
+		valid:    err == nil,
+	}
+	return &rv
+}
+
+func (i *Iterator) SeekFirst() {
+	err := i.iterator.SeekMin()
+	if err != nil {
+		i.valid = false
+		return
+	}
+	if i.curr != nil {
+		i.curr.Close()
+	}
+	i.curr, err = i.iterator.Get()
+	if err != nil {
+		i.valid = false
+	}
+}
+
+func (i *Iterator) Seek(key []byte) {
+	err := i.iterator.Seek(key, forestdb.FDB_ITR_SEEK_HIGHER)
+	if err != nil {
+		i.valid = false
+		return
+	}
+	if i.curr != nil {
+		i.curr.Close()
+	}
+	i.curr, err = i.iterator.Get()
+	if err != nil {
+		i.valid = false
+		return
+	}
+}
+
+func (i *Iterator) Next() {
+	err := i.iterator.Next()
+	if err != nil {
+		i.valid = false
+		return
+	}
+	if i.curr != nil {
+		i.curr.Close()
+	}
+	i.curr, err = i.iterator.Get()
+	if err != nil {
+		i.valid = false
+	}
+}
+
+func (i *Iterator) Current() ([]byte, []byte, bool) {
+	if i.Valid() {
+		return i.Key(), i.Value(), true
+	}
+	return nil, nil, false
+}
+
+func (i *Iterator) Key() []byte {
+	return i.curr.Key()
+}
+
+func (i *Iterator) Value() []byte {
+	return i.curr.Body()
+}
+
+func (i *Iterator) Valid() bool {
+	return i.valid
+}
+
+func (i *Iterator) Close() error {
+	i.valid = false
+	if i.curr != nil {
+		i.curr.Close()
+	}
+	return i.iterator.Close()
+}

vendor/github.com/blevesearch/bleve/index/store/forestdb/reader.go

@@ -0,0 +1,57 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build forestdb
+
+package forestdb
+
+import (
+	"fmt"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/couchbase/goforestdb"
+)
+
+type Reader struct {
+	store    *Store
+	snapshot *forestdb.KVStore
+}
+
+func (r *Reader) BytesSafeAfterClose() bool {
+	return true
+}
+
+func newReader(store *Store) (*Reader, error) {
+	snapshot, err := store.newSnapshot()
+	if err != nil {
+		return nil, fmt.Errorf("error opening snapshot: %v", err)
+	}
+	return &Reader{
+		store:    store,
+		snapshot: snapshot,
+	}, nil
+}
+
+func (r *Reader) Get(key []byte) ([]byte, error) {
+	res, err := r.snapshot.GetKV(key)
+	if err != nil && err != forestdb.RESULT_KEY_NOT_FOUND {
+		return nil, err
+	}
+	return res, nil
+}
+
+func (r *Reader) Iterator(key []byte) store.KVIterator {
+	rv := newIteratorWithSnapshot(r.store, r.snapshot)
+	rv.Seek(key)
+	return rv
+}
+
+func (r *Reader) Close() error {
+	return r.snapshot.Close()
+}

vendor/github.com/blevesearch/bleve/index/store/forestdb/store.go

@@ -0,0 +1,297 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build forestdb
+
+package forestdb
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"sync"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/registry"
+	"github.com/couchbase/goforestdb"
+)
+
+type ForestDBConfig struct {
+	BlockSize                 uint32
+	BufferCacheSize           uint64
+	ChunkSize                 uint16
+	CleanupCacheOnClose       bool
+	CompactionBufferSizeMax   uint32
+	CompactionMinimumFilesize uint64
+	CompactionMode            forestdb.CompactOpt
+	CompactionThreshold       uint8
+	CompactorSleepDuration    uint64
+	CompressDocumentBody      bool
+	DurabilityOpt             forestdb.DurabilityOpt
+	OpenFlags                 forestdb.OpenFlags
+	PurgingInterval           uint32
+	SeqTreeOpt                forestdb.SeqTreeOpt
+	WalFlushBeforeCommit      bool
+	WalThreshold              uint64
+}
+
+const Name = "forestdb"
+
+type Store struct {
+	path     string
+	config   *forestdb.Config
+	kvconfig *forestdb.KVStoreConfig
+	dbfile   *forestdb.File
+	dbkv     *forestdb.KVStore
+	writer   sync.Mutex
+	mo       store.MergeOperator
+}
+
+func New(path string, createIfMissing bool,
+	config map[string]interface{}) (*Store, error) {
+	if config == nil {
+		config = map[string]interface{}{}
+	}
+
+	forestDBDefaultConfig := forestdb.DefaultConfig()
+	forestDBDefaultConfig.SetCompactionMode(forestdb.COMPACT_AUTO)
+	forestDBConfig, err := applyConfig(forestDBDefaultConfig, config)
+	if err != nil {
+		return nil, err
+	}
+
+	rv := Store{
+		path:     path,
+		config:   forestDBConfig,
+		kvconfig: forestdb.DefaultKVStoreConfig(),
+	}
+
+	if createIfMissing {
+		rv.kvconfig.SetCreateIfMissing(true)
+	}
+
+	return &rv, nil
+}
+
+func (s *Store) Open() error {
+	var err error
+	s.dbfile, err = forestdb.Open(s.path, s.config)
+	if err != nil {
+		return err
+	}
+
+	s.dbkv, err = s.dbfile.OpenKVStoreDefault(s.kvconfig)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Store) SetMergeOperator(mo store.MergeOperator) {
+	s.mo = mo
+}
+
+func (s *Store) get(key []byte) ([]byte, error) {
+	res, err := s.dbkv.GetKV(key)
+	if err != nil && err != forestdb.RESULT_KEY_NOT_FOUND {
+		return nil, err
+	}
+	return res, nil
+}
+
+func (s *Store) set(key, val []byte) error {
+	s.writer.Lock()
+	defer s.writer.Unlock()
+	return s.setlocked(key, val)
+}
+
+func (s *Store) setlocked(key, val []byte) error {
+	return s.dbkv.SetKV(key, val)
+}
+
+func (s *Store) delete(key []byte) error {
+	s.writer.Lock()
+	defer s.writer.Unlock()
+	return s.deletelocked(key)
+}
+
+func (s *Store) deletelocked(key []byte) error {
+	return s.dbkv.DeleteKV(key)
+}
+
+func (s *Store) commit() error {
+	return s.dbfile.Commit(forestdb.COMMIT_NORMAL)
+}
+
+func (s *Store) Close() error {
+	err := s.dbkv.Close()
+	if err != nil {
+		return err
+	}
+	return s.dbfile.Close()
+
+}
+
+func (ldbs *Store) iterator(key []byte) store.KVIterator {
+	rv := newIterator(ldbs)
+	rv.Seek(key)
+	return rv
+}
+
+func (s *Store) Reader() (store.KVReader, error) {
+	return newReader(s)
+}
+
+func (ldbs *Store) Writer() (store.KVWriter, error) {
+	return newWriter(ldbs)
+}
+
+func (s *Store) getSeqNum() (forestdb.SeqNum, error) {
+	dbinfo, err := s.dbkv.Info()
+	if err != nil {
+		return 0, err
+	}
+	return dbinfo.LastSeqNum(), nil
+}
+
+func (s *Store) newSnapshot() (*forestdb.KVStore, error) {
+	seqNum, err := s.getSeqNum()
+	if err != nil {
+		return nil, fmt.Errorf("error getting snapshot seqnum: %v", err)
+	}
+	snapshot, err := s.dbkv.SnapshotOpen(seqNum)
+	if err == forestdb.RESULT_NO_DB_INSTANCE {
+		checkAgainSeqNum, err := s.getSeqNum()
+		if err != nil {
+			return nil, fmt.Errorf("error getting snapshot seqnum again: %v", err)
+		}
+		return nil, fmt.Errorf("cannot open snapshot %v, checked again its %v, error: %v", seqNum, checkAgainSeqNum, err)
+	}
+	return snapshot, err
+}
+
+func (s *Store) GetRollbackID() ([]byte, error) {
+	seqNum, err := s.getSeqNum()
+	if err != nil {
+		return nil, err
+	}
+	buf := new(bytes.Buffer)
+	err = binary.Write(buf, binary.LittleEndian, seqNum)
+	if err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func (s *Store) RollbackTo(rollbackId []byte) error {
+	s.writer.Lock()
+	defer s.writer.Unlock()
+	buf := bytes.NewReader(rollbackId)
+	var seqNum forestdb.SeqNum
+	err := binary.Read(buf, binary.LittleEndian, &seqNum)
+	if err != nil {
+		return err
+	}
+	err = s.dbkv.Rollback(seqNum)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify path")
+	}
+	createIfMissing := false
+	cim, ok := config["create_if_missing"].(bool)
+	if ok {
+		createIfMissing = cim
+	}
+	return New(path, createIfMissing, config)
+}
+
+func init() {
+	registry.RegisterKVStore(Name, StoreConstructor)
+}
+
+func applyConfig(c *forestdb.Config, config map[string]interface{}) (
+	*forestdb.Config, error) {
+	v, exists := config["forestDBConfig"]
+	if !exists || v == nil {
+		return c, nil
+	}
+	m, ok := v.(map[string]interface{})
+	if !ok {
+		return c, nil
+	}
+	// These extra steps of json.Marshal()/Unmarshal() help to convert
+	// to the types that we need for the setter calls.
+	b, err := json.Marshal(m)
+	if err != nil {
+		return nil, err
+	}
+	var f ForestDBConfig
+	err = json.Unmarshal(b, &f)
+	if err != nil {
+		return nil, err
+	}
+	if _, exists := m["blockSize"]; exists {
+		c.SetBlockSize(f.BlockSize)
+	}
+	if _, exists := m["bufferCacheSize"]; exists {
+		c.SetBufferCacheSize(f.BufferCacheSize)
+	}
+	if _, exists := m["chunkSize"]; exists {
+		c.SetChunkSize(f.ChunkSize)
+	}
+	if _, exists := m["cleanupCacheOnClose"]; exists {
+		c.SetCleanupCacheOnClose(f.CleanupCacheOnClose)
+	}
+	if _, exists := m["compactionBufferSizeMax"]; exists {
+		c.SetCompactionBufferSizeMax(f.CompactionBufferSizeMax)
+	}
+	if _, exists := m["compactionMinimumFilesize"]; exists {
+		c.SetCompactionMinimumFilesize(f.CompactionMinimumFilesize)
+	}
+	if _, exists := m["compactionMode"]; exists {
+		c.SetCompactionMode(f.CompactionMode)
+	}
+	if _, exists := m["compactionThreshold"]; exists {
+		c.SetCompactionThreshold(f.CompactionThreshold)
+	}
+	if _, exists := m["compactorSleepDuration"]; exists {
+		c.SetCompactorSleepDuration(f.CompactorSleepDuration)
+	}
+	if _, exists := m["compressDocumentBody"]; exists {
+		c.SetCompressDocumentBody(f.CompressDocumentBody)
+	}
+	if _, exists := m["durabilityOpt"]; exists {
+		c.SetDurabilityOpt(f.DurabilityOpt)
+	}
+	if _, exists := m["openFlags"]; exists {
+		c.SetOpenFlags(f.OpenFlags)
+	}
+	if _, exists := m["purgingInterval"]; exists {
+		c.SetPurgingInterval(f.PurgingInterval)
+	}
+	if _, exists := m["seqTreeOpt"]; exists {
+		c.SetSeqTreeOpt(f.SeqTreeOpt)
+	}
+	if _, exists := m["walFlushBeforeCommit"]; exists {
+		c.SetWalFlushBeforeCommit(f.WalFlushBeforeCommit)
+	}
+	if _, exists := m["walThreshold"]; exists {
+		c.SetWalThreshold(f.WalThreshold)
+	}
+	return c, nil
+}

vendor/github.com/blevesearch/bleve/index/store/forestdb/store_test.go

@@ -0,0 +1,636 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build forestdb
+
+package forestdb
+
+import (
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+)
+
+func TestForestDBStore(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("testdir")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	err := os.MkdirAll("testdir", 0700)
+	if err != nil {
+		t.Fatal(err)
+	}
+	s, err := New("testdir/test", true, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := s.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	CommonTestKVStore(t, s)
+}
+
+func TestReaderIsolation(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("testdir")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	err := os.MkdirAll("testdir", 0700)
+	if err != nil {
+		t.Fatal(err)
+	}
+	s, err := New("testdir/test", true, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := s.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	CommonTestReaderIsolation(t, s)
+}
+
+// TestRollbackSameHandle tries to rollback a handle
+// and ensure that subsequent reads from it also
+// reflect the rollback
+func TestRollbackSameHandle(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("testdir")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	err := os.MkdirAll("testdir", 0700)
+	if err != nil {
+		t.Fatal(err)
+	}
+	s, err := New("testdir/test", true, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := s.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	writer, err := s.Writer()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create 2 docs, a and b
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Error(err)
+	}
+
+	err = writer.Set([]byte("b"), []byte("val-b"))
+	if err != nil {
+		t.Error(err)
+	}
+
+	// get the rollback id
+	rollbackId, err := s.GetRollbackID()
+	if err != nil {
+		t.Error(err)
+	}
+
+	// create a 3rd doc c
+	err = writer.Set([]byte("c"), []byte("val-c"))
+	if err != nil {
+		t.Error(err)
+	}
+
+	err = writer.Close()
+	if err != nil {
+		t.Error(err)
+	}
+
+	// make sure c is there
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	val, err := reader.Get([]byte("c"))
+	if err != nil {
+		t.Error(err)
+	}
+	if string(val) != "val-c" {
+		t.Errorf("expected value 'val-c' got '%s'", val)
+	}
+	err = reader.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// now rollback
+	err = s.RollbackTo(rollbackId)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// now make sure c is not there
+	reader, err = s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	val, err = reader.Get([]byte("c"))
+	if err != nil {
+		t.Error(err)
+	}
+	if val != nil {
+		t.Errorf("expected missing, got '%s'", val)
+	}
+	err = reader.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestRollbackNewHandle tries to rollback the
+// database, then opens a new handle, and ensures
+// that the rollback is reflected there as well
+func TestRollbackNewHandle(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("testdir")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	err := os.MkdirAll("testdir", 0700)
+	if err != nil {
+		t.Fatal(err)
+	}
+	s, err := New("testdir/test", true, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := s.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	writer, err := s.Writer()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create 2 docs, a and b
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Error(err)
+	}
+
+	err = writer.Set([]byte("b"), []byte("val-b"))
+	if err != nil {
+		t.Error(err)
+	}
+
+	// get the rollback id
+	rollbackId, err := s.GetRollbackID()
+	if err != nil {
+		t.Error(err)
+	}
+
+	// create a 3rd doc c
+	err = writer.Set([]byte("c"), []byte("val-c"))
+	if err != nil {
+		t.Error(err)
+	}
+
+	err = writer.Close()
+	if err != nil {
+		t.Error(err)
+	}
+
+	// make sure c is there
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	val, err := reader.Get([]byte("c"))
+	if err != nil {
+		t.Error(err)
+	}
+	if string(val) != "val-c" {
+		t.Errorf("expected value 'val-c' got '%s'", val)
+	}
+	err = reader.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// now rollback
+	err = s.RollbackTo(rollbackId)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// now lets open another handle
+	s2, err := New("testdir/test", true, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s2.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s2.Close()
+
+	// now make sure c is not there
+	reader2, err := s2.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	val, err = reader2.Get([]byte("c"))
+	if err != nil {
+		t.Error(err)
+	}
+	if val != nil {
+		t.Errorf("expected missing, got '%s'", val)
+	}
+	err = reader2.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestRollbackOtherHandle tries to create 2 handles
+// at the beginning, then rollback one of them
+// and ensure it affects the other
+func TestRollbackOtherHandle(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("testdir")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	err := os.MkdirAll("testdir", 0700)
+	if err != nil {
+		t.Fatal(err)
+	}
+	s, err := New("testdir/test", true, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := s.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	// open another handle at the same time
+	s2, err := New("testdir/test", true, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s2.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s2.Close()
+
+	writer, err := s.Writer()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create 2 docs, a and b
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Error(err)
+	}
+
+	err = writer.Set([]byte("b"), []byte("val-b"))
+	if err != nil {
+		t.Error(err)
+	}
+
+	// get the rollback id
+	rollbackId, err := s.GetRollbackID()
+	if err != nil {
+		t.Error(err)
+	}
+
+	// create a 3rd doc c
+	err = writer.Set([]byte("c"), []byte("val-c"))
+	if err != nil {
+		t.Error(err)
+	}
+
+	err = writer.Close()
+	if err != nil {
+		t.Error(err)
+	}
+
+	// make sure c is there
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	val, err := reader.Get([]byte("c"))
+	if err != nil {
+		t.Error(err)
+	}
+	if string(val) != "val-c" {
+		t.Errorf("expected value 'val-c' got '%s'", val)
+	}
+	err = reader.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// now rollback
+	err = s.RollbackTo(rollbackId)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// now make sure c is not on the other handle
+	reader2, err := s2.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	val, err = reader2.Get([]byte("c"))
+	if err != nil {
+		t.Error(err)
+	}
+	if val != nil {
+		t.Errorf("expected missing, got '%s'", val)
+	}
+	err = reader2.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func CommonTestKVStore(t *testing.T, s store.KVStore) {
+
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Set([]byte("z"), []byte("val-z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Delete([]byte("z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	batch := writer.NewBatch()
+	batch.Set([]byte("b"), []byte("val-b"))
+	batch.Set([]byte("c"), []byte("val-c"))
+	batch.Set([]byte("d"), []byte("val-d"))
+	batch.Set([]byte("e"), []byte("val-e"))
+	batch.Set([]byte("f"), []byte("val-f"))
+	batch.Set([]byte("g"), []byte("val-g"))
+	batch.Set([]byte("h"), []byte("val-h"))
+	batch.Set([]byte("i"), []byte("val-i"))
+	batch.Set([]byte("j"), []byte("val-j"))
+
+	err = batch.Execute()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	it := reader.Iterator([]byte("b"))
+	key, val, valid := it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "b" {
+		t.Fatalf("expected key b, got %s", key)
+	}
+	if string(val) != "val-b" {
+		t.Fatalf("expected value val-b, got %s", val)
+	}
+
+	it.Next()
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "c" {
+		t.Fatalf("expected key c, got %s", key)
+	}
+	if string(val) != "val-c" {
+		t.Fatalf("expected value val-c, got %s", val)
+	}
+
+	it.Seek([]byte("i"))
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "i" {
+		t.Fatalf("expected key i, got %s", key)
+	}
+	if string(val) != "val-i" {
+		t.Fatalf("expected value val-i, got %s", val)
+	}
+
+	err = it.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
+	// insert a kv pair
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create an isolated reader
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	// verify we see the value already inserted
+	val, err := reader.Get([]byte("a"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-a")) {
+		t.Errorf("expected val-a, got nil")
+	}
+
+	// verify that an iterator sees it
+	count := 0
+	it := reader.Iterator([]byte{0})
+	defer func() {
+		err := it.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it.Valid() {
+		it.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+	// add something after the reader was created
+	writer, err = s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("b"), []byte("val-b"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ensure that a newer reader sees it
+	newReader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := newReader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	val, err = newReader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-b")) {
+		t.Errorf("expected val-b, got nil")
+	}
+
+	// ensure that the director iterator sees it
+	count = 0
+	it2 := newReader.Iterator([]byte{0})
+	defer func() {
+		err := it2.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it2.Valid() {
+		it2.Next()
+		count++
+	}
+	if count != 2 {
+		t.Errorf("expected iterator to see 2, saw %d", count)
+	}
+
+	// but that the isolated reader does not
+	val, err = reader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if val != nil {
+		t.Errorf("expected nil, got %v", val)
+	}
+
+	// and ensure that the iterator on the isolated reader also does not
+	count = 0
+	it3 := reader.Iterator([]byte{0})
+	defer func() {
+		err := it3.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it3.Valid() {
+		it3.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+}

vendor/github.com/blevesearch/bleve/index/store/forestdb/writer.go

@@ -0,0 +1,71 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build forestdb
+
+package forestdb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+)
+
+type Writer struct {
+	store *Store
+}
+
+func (w *Writer) BytesSafeAfterClose() bool {
+	return true
+}
+
+func newWriter(store *Store) (*Writer, error) {
+	store.writer.Lock()
+	return &Writer{
+		store: store,
+	}, nil
+}
+
+func (w *Writer) Set(key, val []byte) error {
+	err := w.store.setlocked(key, val)
+	if err != nil {
+		return err
+	}
+	return w.store.commit()
+}
+
+func (w *Writer) Delete(key []byte) error {
+	err := w.store.deletelocked(key)
+	if err != nil {
+		return err
+	}
+	return w.store.commit()
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	return &Batch{
+		s:      w.store,
+		ops:    make([]op, 0, 1000),
+		merges: make(map[string][][]byte),
+	}
+}
+
+func (w *Writer) Close() error {
+	w.store.writer.Unlock()
+	return nil
+}
+
+// these two methods can safely read using the regular
+// methods without a read transaction, because we know
+// that no one else is writing but us
+func (w *Writer) Get(key []byte) ([]byte, error) {
+	return w.store.get(key)
+}
+
+func (w *Writer) Iterator(key []byte) store.KVIterator {
+	return w.store.iterator(key)
+}

vendor/github.com/blevesearch/bleve/index/store/goleveldb/batch.go

@@ -0,0 +1,53 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package goleveldb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type Batch struct {
+	w     *Writer
+	merge *store.EmulatedMerge
+	batch *leveldb.Batch
+}
+
+func (b *Batch) Set(key, val []byte) {
+	b.batch.Put(key, val)
+}
+
+func (b *Batch) Delete(key []byte) {
+	b.batch.Delete(key)
+}
+
+func (b *Batch) Merge(key, val []byte) {
+	b.merge.Merge(key, val)
+}
+
+func (b *Batch) Execute() error {
+
+	// first process merges
+	ops, err := b.merge.ExecuteDeferred(b.w)
+	if err != nil {
+		return err
+	}
+	for _, op := range ops {
+		b.batch.Put(op.K, op.V)
+	}
+
+	wopts := defaultWriteOptions()
+	err = b.w.store.db.Write(b.batch, wopts)
+	return err
+}
+
+func (b *Batch) Close() error {
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/goleveldb/iterator.go

@@ -0,0 +1,75 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package goleveldb
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+)
+
+type Iterator struct {
+	store    *Store
+	iterator iterator.Iterator
+}
+
+func newIterator(store *Store) *Iterator {
+	ropts := defaultReadOptions()
+	iter := store.db.NewIterator(nil, ropts)
+	rv := Iterator{
+		store:    store,
+		iterator: iter,
+	}
+	return &rv
+}
+
+func newIteratorWithSnapshot(store *Store, snapshot *leveldb.Snapshot) *Iterator {
+	options := defaultReadOptions()
+	iter := snapshot.NewIterator(nil, options)
+	rv := Iterator{
+		store:    store,
+		iterator: iter,
+	}
+	return &rv
+}
+
+func (ldi *Iterator) SeekFirst() {
+	ldi.iterator.First()
+}
+
+func (ldi *Iterator) Seek(key []byte) {
+	ldi.iterator.Seek(key)
+}
+
+func (ldi *Iterator) Next() {
+	ldi.iterator.Next()
+}
+
+func (ldi *Iterator) Current() ([]byte, []byte, bool) {
+	if ldi.Valid() {
+		return ldi.Key(), ldi.Value(), true
+	}
+	return nil, nil, false
+}
+
+func (ldi *Iterator) Key() []byte {
+	return ldi.iterator.Key()
+}
+
+func (ldi *Iterator) Value() []byte {
+	return ldi.iterator.Value()
+}
+
+func (ldi *Iterator) Valid() bool {
+	return ldi.iterator.Valid()
+}
+
+func (ldi *Iterator) Close() error {
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/goleveldb/reader.go

@@ -0,0 +1,47 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package goleveldb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type Reader struct {
+	store    *Store
+	snapshot *leveldb.Snapshot
+}
+
+func newReader(store *Store) (*Reader, error) {
+	snapshot, _ := store.db.GetSnapshot()
+	return &Reader{
+		store:    store,
+		snapshot: snapshot,
+	}, nil
+}
+
+func (r *Reader) BytesSafeAfterClose() bool {
+	return false
+}
+
+func (r *Reader) Get(key []byte) ([]byte, error) {
+	return r.store.getWithSnapshot(key, r.snapshot)
+}
+
+func (r *Reader) Iterator(key []byte) store.KVIterator {
+	rv := newIteratorWithSnapshot(r.store, r.snapshot)
+	rv.Seek(key)
+	return rv
+}
+
+func (r *Reader) Close() error {
+	r.snapshot.Release()
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/goleveldb/store.go

@@ -0,0 +1,172 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package goleveldb
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/registry"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+const Name = "goleveldb"
+
+type Store struct {
+	path   string
+	opts   *opt.Options
+	db     *leveldb.DB
+	writer sync.Mutex
+	mo     store.MergeOperator
+}
+
+func New(path string, config map[string]interface{}) (*Store, error) {
+	rv := Store{
+		path: path,
+		opts: &opt.Options{},
+	}
+
+	_, err := applyConfig(rv.opts, config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &rv, nil
+}
+
+func (ldbs *Store) Open() error {
+	var err error
+	ldbs.db, err = leveldb.OpenFile(ldbs.path, ldbs.opts)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ldbs *Store) SetMergeOperator(mo store.MergeOperator) {
+	ldbs.mo = mo
+}
+
+func (ldbs *Store) get(key []byte) ([]byte, error) {
+	options := defaultReadOptions()
+	b, err := ldbs.db.Get(key, options)
+	if err == leveldb.ErrNotFound {
+		return nil, nil
+	}
+	return b, err
+}
+
+func (ldbs *Store) getWithSnapshot(key []byte, snapshot *leveldb.Snapshot) ([]byte, error) {
+	options := defaultReadOptions()
+	b, err := snapshot.Get(key, options)
+	if err == leveldb.ErrNotFound {
+		return nil, nil
+	}
+	return b, err
+}
+
+func (ldbs *Store) set(key, val []byte) error {
+	ldbs.writer.Lock()
+	defer ldbs.writer.Unlock()
+	return ldbs.setlocked(key, val)
+}
+
+func (ldbs *Store) setlocked(key, val []byte) error {
+	options := defaultWriteOptions()
+	err := ldbs.db.Put(key, val, options)
+	return err
+}
+
+func (ldbs *Store) delete(key []byte) error {
+	ldbs.writer.Lock()
+	defer ldbs.writer.Unlock()
+	return ldbs.deletelocked(key)
+}
+
+func (ldbs *Store) deletelocked(key []byte) error {
+	options := defaultWriteOptions()
+	err := ldbs.db.Delete(key, options)
+	return err
+}
+
+func (ldbs *Store) Close() error {
+	return ldbs.db.Close()
+}
+
+func (ldbs *Store) iterator(key []byte) store.KVIterator {
+	rv := newIterator(ldbs)
+	rv.Seek(key)
+	return rv
+}
+
+func (ldbs *Store) Reader() (store.KVReader, error) {
+	return newReader(ldbs)
+}
+
+func (ldbs *Store) Writer() (store.KVWriter, error) {
+	return newWriter(ldbs)
+}
+
+func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify path")
+	}
+	return New(path, config)
+}
+
+func init() {
+	registry.RegisterKVStore(Name, StoreConstructor)
+}
+
+func applyConfig(o *opt.Options, config map[string]interface{}) (
+	*opt.Options, error) {
+
+	cim, ok := config["create_if_missing"].(bool)
+	if ok {
+		o.ErrorIfMissing = !cim
+	}
+
+	eie, ok := config["error_if_exists"].(bool)
+	if ok {
+		o.ErrorIfExist = eie
+	}
+
+	wbs, ok := config["write_buffer_size"].(float64)
+	if ok {
+		o.WriteBuffer = int(wbs)
+	}
+
+	bs, ok := config["block_size"].(float64)
+	if ok {
+		o.BlockSize = int(bs)
+	}
+
+	bri, ok := config["block_restart_interval"].(float64)
+	if ok {
+		o.BlockRestartInterval = int(bri)
+	}
+
+	lcc, ok := config["lru_cache_capacity"].(float64)
+	if ok {
+		o.BlockCacheCapacity = int(lcc)
+	}
+
+	bfbpk, ok := config["bloom_filter_bits_per_key"].(float64)
+	if ok {
+		bf := filter.NewBloomFilter(int(bfbpk))
+		o.Filter = bf
+	}
+
+	return o, nil
+}

vendor/github.com/blevesearch/bleve/index/store/goleveldb/store_test.go

@@ -0,0 +1,296 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package goleveldb
+
+import (
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+)
+
+var leveldbTestOptions = map[string]interface{}{
+	"create_if_missing": true,
+}
+
+func TestLevelDBStore(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	s, err := New("test", leveldbTestOptions)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := s.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	CommonTestKVStore(t, s)
+}
+
+func TestReaderIsolation(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	s, err := New("test", leveldbTestOptions)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := s.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	CommonTestReaderIsolation(t, s)
+}
+
+func CommonTestKVStore(t *testing.T, s store.KVStore) {
+
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Set([]byte("z"), []byte("val-z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Delete([]byte("z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	batch := writer.NewBatch()
+	batch.Set([]byte("b"), []byte("val-b"))
+	batch.Set([]byte("c"), []byte("val-c"))
+	batch.Set([]byte("d"), []byte("val-d"))
+	batch.Set([]byte("e"), []byte("val-e"))
+	batch.Set([]byte("f"), []byte("val-f"))
+	batch.Set([]byte("g"), []byte("val-g"))
+	batch.Set([]byte("h"), []byte("val-h"))
+	batch.Set([]byte("i"), []byte("val-i"))
+	batch.Set([]byte("j"), []byte("val-j"))
+
+	err = batch.Execute()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	it := reader.Iterator([]byte("b"))
+	key, val, valid := it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "b" {
+		t.Fatalf("expected key b, got %s", key)
+	}
+	if string(val) != "val-b" {
+		t.Fatalf("expected value val-b, got %s", val)
+	}
+
+	it.Next()
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "c" {
+		t.Fatalf("expected key c, got %s", key)
+	}
+	if string(val) != "val-c" {
+		t.Fatalf("expected value val-c, got %s", val)
+	}
+
+	it.Seek([]byte("i"))
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "i" {
+		t.Fatalf("expected key i, got %s", key)
+	}
+	if string(val) != "val-i" {
+		t.Fatalf("expected value val-i, got %s", val)
+	}
+
+	err = it.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
+	// insert a kv pair
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create an isolated reader
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	// verify that we see the value already inserted
+	val, err := reader.Get([]byte("a"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-a")) {
+		t.Errorf("expected val-a, got nil")
+	}
+
+	// verify that an iterator sees it
+	count := 0
+	it := reader.Iterator([]byte{0})
+	defer func() {
+		err := it.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it.Valid() {
+		it.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+	// add something after the reader was created
+	writer, err = s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("b"), []byte("val-b"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ensure that a newer reader sees it
+	newReader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := newReader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	val, err = newReader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-b")) {
+		t.Errorf("expected val-b, got nil")
+	}
+
+	// ensure that the director iterator sees it
+	count = 0
+	it = newReader.Iterator([]byte{0})
+	defer func() {
+		err := it.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it.Valid() {
+		it.Next()
+		count++
+	}
+	if count != 2 {
+		t.Errorf("expected iterator to see 2, saw %d", count)
+	}
+
+	// but that the isolated reader does not
+	val, err = reader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if val != nil {
+		t.Errorf("expected nil, got %v", val)
+	}
+
+	// and ensure that the iterator on the isolated reader also does not
+	count = 0
+	it = reader.Iterator([]byte{0})
+	defer func() {
+		err := it.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it.Valid() {
+		it.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+}

vendor/github.com/blevesearch/bleve/index/store/goleveldb/util.go

@@ -0,0 +1,26 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package goleveldb
+
+import (
+	"github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+func defaultWriteOptions() *opt.WriteOptions {
+	wo := &opt.WriteOptions{}
+	// request fsync on write for safety
+	wo.Sync = true
+	return wo
+}
+
+func defaultReadOptions() *opt.ReadOptions {
+	ro := &opt.ReadOptions{}
+	return ro
+}

vendor/github.com/blevesearch/bleve/index/store/goleveldb/writer.go

@@ -0,0 +1,63 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package goleveldb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type Writer struct {
+	store *Store
+}
+
+func newWriter(store *Store) (*Writer, error) {
+	store.writer.Lock()
+	return &Writer{
+		store: store,
+	}, nil
+}
+
+func (w *Writer) BytesSafeAfterClose() bool {
+	return false
+}
+
+func (w *Writer) Set(key, val []byte) error {
+	return w.store.setlocked(key, val)
+}
+
+func (w *Writer) Delete(key []byte) error {
+	return w.store.deletelocked(key)
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	rv := Batch{
+		w:     w,
+		merge: store.NewEmulatedMerge(w.store.mo),
+		batch: new(leveldb.Batch),
+	}
+	return &rv
+}
+
+func (w *Writer) Close() error {
+	w.store.writer.Unlock()
+	return nil
+}
+
+// these two methods can safely read using the regular
+// methods without a read transaction, because we know
+// that no one else is writing but us
+func (w *Writer) Get(key []byte) ([]byte, error) {
+	return w.store.get(key)
+}
+
+func (w *Writer) Iterator(key []byte) store.KVIterator {
+	return w.store.iterator(key)
+}

vendor/github.com/blevesearch/bleve/index/store/gorocksdb/batch.go

@@ -0,0 +1,43 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build rocksdb
+
+package rocksdb
+
+import (
+	"github.com/tecbot/gorocksdb"
+)
+
+type Batch struct {
+	w     *Writer
+	batch *gorocksdb.WriteBatch
+}
+
+func (b *Batch) Set(key, val []byte) {
+	b.batch.Put(key, val)
+}
+
+func (b *Batch) Delete(key []byte) {
+	b.batch.Delete(key)
+}
+
+func (b *Batch) Merge(key, val []byte) {
+	b.batch.Merge(key, val)
+}
+
+func (b *Batch) Execute() error {
+	wopts := defaultWriteOptions()
+	err := b.w.store.db.Write(wopts, b.batch)
+	return err
+}
+
+func (b *Batch) Close() error {
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/gorocksdb/iterator.go

@@ -0,0 +1,76 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build rocksdb
+
+package rocksdb
+
+import (
+	"github.com/tecbot/gorocksdb"
+)
+
+type Iterator struct {
+	store    *Store
+	iterator *gorocksdb.Iterator
+}
+
+func newIterator(store *Store) *Iterator {
+	ropts := defaultReadOptions()
+	rv := Iterator{
+		store:    store,
+		iterator: store.db.NewIterator(ropts),
+	}
+	return &rv
+}
+
+func newIteratorWithSnapshot(store *Store, snapshot *gorocksdb.Snapshot) *Iterator {
+	options := defaultReadOptions()
+	options.SetSnapshot(snapshot)
+	rv := Iterator{
+		store:    store,
+		iterator: store.db.NewIterator(options),
+	}
+	return &rv
+}
+
+func (ldi *Iterator) SeekFirst() {
+	ldi.iterator.SeekToFirst()
+}
+
+func (ldi *Iterator) Seek(key []byte) {
+	ldi.iterator.Seek(key)
+}
+
+func (ldi *Iterator) Next() {
+	ldi.iterator.Next()
+}
+
+func (ldi *Iterator) Current() ([]byte, []byte, bool) {
+	if ldi.Valid() {
+		return ldi.Key(), ldi.Value(), true
+	}
+	return nil, nil, false
+}
+
+func (ldi *Iterator) Key() []byte {
+	return ldi.iterator.Key().Data()
+}
+
+func (ldi *Iterator) Value() []byte {
+	return ldi.iterator.Value().Data()
+}
+
+func (ldi *Iterator) Valid() bool {
+	return ldi.iterator.Valid()
+}
+
+func (ldi *Iterator) Close() error {
+	ldi.iterator.Close()
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/gorocksdb/reader.go

@@ -0,0 +1,48 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build rocksdb
+
+package rocksdb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/tecbot/gorocksdb"
+)
+
+type Reader struct {
+	store    *Store
+	snapshot *gorocksdb.Snapshot
+}
+
+func newReader(store *Store) (*Reader, error) {
+	return &Reader{
+		store:    store,
+		snapshot: store.db.NewSnapshot(),
+	}, nil
+}
+
+func (r *Reader) BytesSafeAfterClose() bool {
+	return false
+}
+
+func (r *Reader) Get(key []byte) ([]byte, error) {
+	return r.store.getWithSnapshot(key, r.snapshot)
+}
+
+func (r *Reader) Iterator(key []byte) store.KVIterator {
+	rv := newIteratorWithSnapshot(r.store, r.snapshot)
+	rv.Seek(key)
+	return rv
+}
+
+func (r *Reader) Close() error {
+	r.snapshot.Release()
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/gorocksdb/store.go

@@ -0,0 +1,146 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build rocksdb
+
+package rocksdb
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/registry"
+	"github.com/tecbot/gorocksdb"
+)
+
+const Name = "rocksdb"
+
+type Store struct {
+	path   string
+	opts   *gorocksdb.Options
+	db     *gorocksdb.DB
+	writer sync.Mutex
+}
+
+func New(path string, config map[string]interface{}) (*Store, error) {
+	rv := Store{
+		path: path,
+		opts: gorocksdb.NewDefaultOptions(),
+	}
+
+	_, err := applyConfig(rv.opts, config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &rv, nil
+}
+
+func (ldbs *Store) Open() error {
+	var err error
+	ldbs.db, err = gorocksdb.OpenDb(ldbs.opts, ldbs.path)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ldbs *Store) SetMergeOperator(mo store.MergeOperator) {
+	ldbs.opts.SetMergeOperator(mo)
+}
+
+func (ldbs *Store) get(key []byte) ([]byte, error) {
+	options := defaultReadOptions()
+	b, err := ldbs.db.Get(options, key)
+	return b.Data(), err
+}
+
+func (ldbs *Store) getWithSnapshot(key []byte, snapshot *gorocksdb.Snapshot) ([]byte, error) {
+	options := defaultReadOptions()
+	options.SetSnapshot(snapshot)
+	b, err := ldbs.db.Get(options, key)
+	return b.Data(), err
+}
+
+func (ldbs *Store) set(key, val []byte) error {
+	ldbs.writer.Lock()
+	defer ldbs.writer.Unlock()
+	return ldbs.setlocked(key, val)
+}
+
+func (ldbs *Store) setlocked(key, val []byte) error {
+	options := defaultWriteOptions()
+	err := ldbs.db.Put(options, key, val)
+	return err
+}
+
+func (ldbs *Store) delete(key []byte) error {
+	ldbs.writer.Lock()
+	defer ldbs.writer.Unlock()
+	return ldbs.deletelocked(key)
+}
+
+func (ldbs *Store) deletelocked(key []byte) error {
+	options := defaultWriteOptions()
+	err := ldbs.db.Delete(options, key)
+	return err
+}
+
+func (ldbs *Store) Close() error {
+	ldbs.db.Close()
+	return nil
+}
+
+func (ldbs *Store) iterator(key []byte) store.KVIterator {
+	rv := newIterator(ldbs)
+	rv.Seek(key)
+	return rv
+}
+
+func (ldbs *Store) Reader() (store.KVReader, error) {
+	return newReader(ldbs)
+}
+
+func (ldbs *Store) Writer() (store.KVWriter, error) {
+	return newWriter(ldbs)
+}
+
+func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify path")
+	}
+	return New(path, config)
+}
+
+func init() {
+	registry.RegisterKVStore(Name, StoreConstructor)
+}
+
+func applyConfig(o *gorocksdb.Options, config map[string]interface{}) (
+	*gorocksdb.Options, error) {
+
+	cim, ok := config["create_if_missing"].(bool)
+	if ok {
+		o.SetCreateIfMissing(cim)
+	}
+
+	eie, ok := config["error_if_exists"].(bool)
+	if ok {
+		o.SetErrorIfExists(eie)
+	}
+
+	wbs, ok := config["write_buffer_size"].(float64)
+	if ok {
+		o.SetWriteBufferSize(int(wbs))
+	}
+
+	return o, nil
+}

vendor/github.com/blevesearch/bleve/index/store/gorocksdb/store_test.go

@@ -0,0 +1,298 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build rocksdb
+
+package rocksdb
+
+import (
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+)
+
+var rocksdbTestOptions = map[string]interface{}{
+	"create_if_missing": true,
+}
+
+func TestGoRocksDBStore(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	s, err := New("test", rocksdbTestOptions)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := s.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	CommonTestKVStore(t, s)
+}
+
+func TestReaderIsolation(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	s, err := New("test", rocksdbTestOptions)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := s.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	CommonTestReaderIsolation(t, s)
+}
+
+func CommonTestKVStore(t *testing.T, s store.KVStore) {
+
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Set([]byte("z"), []byte("val-z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Delete([]byte("z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	batch := writer.NewBatch()
+	batch.Set([]byte("b"), []byte("val-b"))
+	batch.Set([]byte("c"), []byte("val-c"))
+	batch.Set([]byte("d"), []byte("val-d"))
+	batch.Set([]byte("e"), []byte("val-e"))
+	batch.Set([]byte("f"), []byte("val-f"))
+	batch.Set([]byte("g"), []byte("val-g"))
+	batch.Set([]byte("h"), []byte("val-h"))
+	batch.Set([]byte("i"), []byte("val-i"))
+	batch.Set([]byte("j"), []byte("val-j"))
+
+	err = batch.Execute()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	it := reader.Iterator([]byte("b"))
+	key, val, valid := it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "b" {
+		t.Fatalf("expected key b, got %s", key)
+	}
+	if string(val) != "val-b" {
+		t.Fatalf("expected value val-b, got %s", val)
+	}
+
+	it.Next()
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "c" {
+		t.Fatalf("expected key c, got %s", key)
+	}
+	if string(val) != "val-c" {
+		t.Fatalf("expected value val-c, got %s", val)
+	}
+
+	it.Seek([]byte("i"))
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "i" {
+		t.Fatalf("expected key i, got %s", key)
+	}
+	if string(val) != "val-i" {
+		t.Fatalf("expected value val-i, got %s", val)
+	}
+
+	err = it.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
+	// insert a kv pair
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create an isolated reader
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	// verify that we see the value already inserted
+	val, err := reader.Get([]byte("a"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-a")) {
+		t.Errorf("expected val-a, got nil")
+	}
+
+	// verify that an iterator sees it
+	count := 0
+	it := reader.Iterator([]byte{0})
+	defer func() {
+		err := it.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it.Valid() {
+		it.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+	// add something after the reader was created
+	writer, err = s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("b"), []byte("val-b"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ensure that a newer reader sees it
+	newReader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := newReader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	val, err = newReader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-b")) {
+		t.Errorf("expected val-b, got nil")
+	}
+
+	// ensure that the director iterator sees it
+	count = 0
+	it2 := newReader.Iterator([]byte{0})
+	defer func() {
+		err := it2.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it2.Valid() {
+		it2.Next()
+		count++
+	}
+	if count != 2 {
+		t.Errorf("expected iterator to see 2, saw %d", count)
+	}
+
+	// but that the isolated reader does not
+	val, err = reader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if val != nil {
+		t.Errorf("expected nil, got %v", val)
+	}
+
+	// and ensure that the iterator on the isolated reader also does not
+	count = 0
+	it3 := reader.Iterator([]byte{0})
+	defer func() {
+		err := it3.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it3.Valid() {
+		it3.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+}

vendor/github.com/blevesearch/bleve/index/store/gorocksdb/util.go

@@ -0,0 +1,28 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build rocksdb
+
+package rocksdb
+
+import (
+	"github.com/tecbot/gorocksdb"
+)
+
+func defaultWriteOptions() *gorocksdb.WriteOptions {
+	wo := gorocksdb.NewDefaultWriteOptions()
+	// request fsync on write for safety
+	wo.SetSync(true)
+	return wo
+}
+
+func defaultReadOptions() *gorocksdb.ReadOptions {
+	ro := gorocksdb.NewDefaultReadOptions()
+	return ro
+}

vendor/github.com/blevesearch/bleve/index/store/gorocksdb/writer.go

@@ -0,0 +1,64 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build rocksdb
+
+package rocksdb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/tecbot/gorocksdb"
+)
+
+type Writer struct {
+	store *Store
+}
+
+func newWriter(store *Store) (*Writer, error) {
+	store.writer.Lock()
+	return &Writer{
+		store: store,
+	}, nil
+}
+
+func (w *Writer) BytesSafeAfterClose() bool {
+	return false
+}
+
+func (w *Writer) Set(key, val []byte) error {
+	return w.store.setlocked(key, val)
+}
+
+func (w *Writer) Delete(key []byte) error {
+	return w.store.deletelocked(key)
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	rv := Batch{
+		w:     w,
+		batch: gorocksdb.NewWriteBatch(),
+	}
+	return &rv
+}
+
+func (w *Writer) Close() error {
+	w.store.writer.Unlock()
+	return nil
+}
+
+// these two methods can safely read using the regular
+// methods without a read transaction, because we know
+// that no one else is writing but us
+func (w *Writer) Get(key []byte) ([]byte, error) {
+	return w.store.get(key)
+}
+
+func (w *Writer) Iterator(key []byte) store.KVIterator {
+	return w.store.iterator(key)
+}

vendor/github.com/blevesearch/bleve/index/store/gtreap/gtreap.go

@@ -0,0 +1,96 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the
+//  License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing,
+//  software distributed under the License is distributed on an "AS
+//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+//  express or implied. See the License for the specific language
+//  governing permissions and limitations under the License.
+
+// Package gtreap provides an in-memory implementation of the
+// KVStore interfaces using the gtreap balanced-binary treap,
+// copy-on-write data structure.
+package gtreap
+
+import (
+	"bytes"
+	"fmt"
+	"sync"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/registry"
+
+	"github.com/steveyen/gtreap"
+)
+
+const Name = "gtreap"
+
+func init() {
+	registry.RegisterKVStore(Name, StoreConstructor)
+}
+
+const MAX_CONCURRENT_WRITERS = 1
+
+func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
+	s := &Store{
+		availableWriters: make(chan bool, MAX_CONCURRENT_WRITERS),
+		t:                gtreap.NewTreap(itemCompare),
+	}
+	for i := 0; i < MAX_CONCURRENT_WRITERS; i++ {
+		s.availableWriters <- true
+	}
+	return s, nil
+}
+
+type Item struct {
+	k []byte
+	v []byte
+}
+
+func itemCompare(a, b interface{}) int {
+	return bytes.Compare(a.(*Item).k, b.(*Item).k)
+}
+
+type Store struct {
+	availableWriters chan bool
+
+	m sync.Mutex
+	t *gtreap.Treap
+
+	mo store.MergeOperator
+}
+
+type Writer struct {
+	s *Store
+}
+
+func (s *Store) Open() error {
+	return nil
+}
+
+func (s *Store) SetMergeOperator(mo store.MergeOperator) {
+	s.mo = mo
+}
+
+func (s *Store) Close() error {
+	close(s.availableWriters)
+	return nil
+}
+
+func (s *Store) Reader() (store.KVReader, error) {
+	s.m.Lock()
+	t := s.t
+	s.m.Unlock()
+	return &Reader{t: t}, nil
+}
+
+func (s *Store) Writer() (store.KVWriter, error) {
+	available, ok := <-s.availableWriters
+	if !ok || !available {
+		return nil, fmt.Errorf("no available writers")
+	}
+
+	return &Writer{s: s}, nil
+}

vendor/github.com/blevesearch/bleve/index/store/gtreap/gtreap_test.go

@@ -0,0 +1,259 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the
+//  License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing,
+//  software distributed under the License is distributed on an "AS
+//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+//  express or implied. See the License for the specific language
+//  governing permissions and limitations under the License.
+
+package gtreap
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+)
+
+func TestGTreapStore(t *testing.T) {
+	s, err := StoreConstructor(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	CommonTestKVStore(t, s)
+}
+
+func TestReaderIsolation(t *testing.T) {
+	s, err := StoreConstructor(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	CommonTestReaderIsolation(t, s)
+}
+
+func CommonTestKVStore(t *testing.T, s store.KVStore) {
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Set([]byte("z"), []byte("val-z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Delete([]byte("z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	batch := writer.NewBatch()
+	batch.Set([]byte("b"), []byte("val-b"))
+	batch.Set([]byte("c"), []byte("val-c"))
+	batch.Set([]byte("d"), []byte("val-d"))
+	batch.Set([]byte("e"), []byte("val-e"))
+	batch.Set([]byte("f"), []byte("val-f"))
+	batch.Set([]byte("g"), []byte("val-g"))
+	batch.Set([]byte("h"), []byte("val-h"))
+	batch.Set([]byte("i"), []byte("val-i"))
+	batch.Set([]byte("j"), []byte("val-j"))
+
+	err = batch.Execute()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	it := reader.Iterator([]byte("b"))
+	key, val, valid := it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "b" {
+		t.Fatalf("expected key b, got %s", key)
+	}
+	if string(val) != "val-b" {
+		t.Fatalf("expected value val-b, got %s", val)
+	}
+
+	it.Next()
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "c" {
+		t.Fatalf("expected key c, got %s", key)
+	}
+	if string(val) != "val-c" {
+		t.Fatalf("expected value val-c, got %s", val)
+	}
+
+	it.Seek([]byte("i"))
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "i" {
+		t.Fatalf("expected key i, got %s", key)
+	}
+	if string(val) != "val-i" {
+		t.Fatalf("expected value val-i, got %s", val)
+	}
+
+	err = it.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
+	// insert a kv pair
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create an isolated reader
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	// verify that we see the value already inserted
+	val, err := reader.Get([]byte("a"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-a")) {
+		t.Errorf("expected val-a, got nil")
+	}
+
+	// verify that an iterator sees it
+	count := 0
+	it := reader.Iterator([]byte{0})
+	defer func() {
+		err := it.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it.Valid() {
+		it.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+	// add something after the reader was created
+	writer, err = s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	valB := []byte("val-b")
+	err = writer.Set([]byte("b"), valB)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ensure that a newer reader sees it
+	newReader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := newReader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	val, err = newReader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-b")) {
+		t.Errorf("expected val-b, got %s", val)
+	}
+
+	// ensure that the director iterator sees it
+	count = 0
+	it2 := newReader.Iterator([]byte{0})
+	defer func() {
+		err := it2.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it2.Valid() {
+		it2.Next()
+		count++
+	}
+	if count != 2 {
+		t.Errorf("expected iterator to see 2, saw %d", count)
+	}
+
+	// but that the isolated reader does not
+	val, err = reader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if val != nil {
+		t.Errorf("expected nil, got %v", val)
+	}
+
+	// and ensure that the iterator on the isolated reader also does not
+	count = 0
+	it3 := reader.Iterator([]byte{0})
+	defer func() {
+		err := it3.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it3.Valid() {
+		it3.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+}

vendor/github.com/blevesearch/bleve/index/store/gtreap/iterator.go

@@ -0,0 +1,132 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the
+//  License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing,
+//  software distributed under the License is distributed on an "AS
+//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+//  express or implied. See the License for the specific language
+//  governing permissions and limitations under the License.
+
+// Package gtreap provides an in-memory implementation of the
+// KVStore interfaces using the gtreap balanced-binary treap,
+// copy-on-write data structure.
+package gtreap
+
+import (
+	"sync"
+
+	"github.com/steveyen/gtreap"
+)
+
+type Iterator struct {
+	t *gtreap.Treap
+
+	m        sync.Mutex
+	cancelCh chan struct{}
+	nextCh   chan *Item
+	curr     *Item
+	currOk   bool
+}
+
+func newIterator(t *gtreap.Treap) *Iterator {
+	return &Iterator{t: t}
+}
+
+func (w *Iterator) SeekFirst() {
+	min := w.t.Min()
+	if min != nil {
+		w.restart(min.(*Item))
+	} else {
+		w.restart(nil)
+	}
+}
+
+func (w *Iterator) Seek(k []byte) {
+	w.restart(&Item{k: k})
+}
+
+func (w *Iterator) restart(start *Item) *Iterator {
+	cancelCh := make(chan struct{})
+	nextCh := make(chan *Item, 1)
+
+	w.m.Lock()
+	if w.cancelCh != nil {
+		close(w.cancelCh)
+	}
+	w.cancelCh = cancelCh
+	w.nextCh = nextCh
+	w.curr = nil
+	w.currOk = false
+	w.m.Unlock()
+
+	go func() {
+		if start != nil {
+			w.t.VisitAscend(start, func(itm gtreap.Item) bool {
+				select {
+				case <-cancelCh:
+					return false
+				case nextCh <- itm.(*Item):
+					return true
+				}
+			})
+		}
+		close(nextCh)
+	}()
+
+	w.Next()
+
+	return w
+}
+
+func (w *Iterator) Next() {
+	w.m.Lock()
+	nextCh := w.nextCh
+	w.m.Unlock()
+	w.curr, w.currOk = <-nextCh
+}
+
+func (w *Iterator) Current() ([]byte, []byte, bool) {
+	w.m.Lock()
+	defer w.m.Unlock()
+	if !w.currOk || w.curr == nil {
+		return nil, nil, false
+	}
+	return w.curr.k, w.curr.v, w.currOk
+}
+
+func (w *Iterator) Key() []byte {
+	k, _, ok := w.Current()
+	if !ok {
+		return nil
+	}
+	return k
+}
+
+func (w *Iterator) Value() []byte {
+	_, v, ok := w.Current()
+	if !ok {
+		return nil
+	}
+	return v
+}
+
+func (w *Iterator) Valid() bool {
+	_, _, ok := w.Current()
+	return ok
+}
+
+func (w *Iterator) Close() error {
+	w.m.Lock()
+	if w.cancelCh != nil {
+		close(w.cancelCh)
+	}
+	w.cancelCh = nil
+	w.nextCh = nil
+	w.curr = nil
+	w.currOk = false
+	w.m.Unlock()
+
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/gtreap/reader.go

@@ -0,0 +1,45 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the
+//  License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing,
+//  software distributed under the License is distributed on an "AS
+//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+//  express or implied. See the License for the specific language
+//  governing permissions and limitations under the License.
+
+// Package gtreap provides an in-memory implementation of the
+// KVStore interfaces using the gtreap balanced-binary treap,
+// copy-on-write data structure.
+package gtreap
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+
+	"github.com/steveyen/gtreap"
+)
+
+type Reader struct {
+	t *gtreap.Treap
+}
+
+func (w *Reader) BytesSafeAfterClose() bool {
+	return false
+}
+
+func (w *Reader) Get(k []byte) (v []byte, err error) {
+	itm := w.t.Get(&Item{k: k})
+	if itm != nil {
+		return itm.(*Item).v, nil
+	}
+	return nil, nil
+}
+
+func (w *Reader) Iterator(k []byte) store.KVIterator {
+	return newIterator(w.t).restart(&Item{k: k})
+}
+
+func (w *Reader) Close() error {
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/gtreap/writer.go

@@ -0,0 +1,72 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the
+//  License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing,
+//  software distributed under the License is distributed on an "AS
+//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+//  express or implied. See the License for the specific language
+//  governing permissions and limitations under the License.
+
+// Package gtreap provides an in-memory implementation of the
+// KVStore interfaces using the gtreap balanced-binary treap,
+// copy-on-write data structure.
+package gtreap
+
+import (
+	"math/rand"
+
+	"github.com/blevesearch/bleve/index/store"
+)
+
+func (w *Writer) BytesSafeAfterClose() bool {
+	return false
+}
+
+func (w *Writer) Get(k []byte) (v []byte, err error) {
+	w.s.m.Lock()
+	t := w.s.t
+	w.s.m.Unlock()
+
+	itm := t.Get(&Item{k: k})
+	if itm != nil {
+		return itm.(*Item).v, nil
+	}
+	return nil, nil
+}
+
+func (w *Writer) Iterator(k []byte) store.KVIterator {
+	w.s.m.Lock()
+	t := w.s.t
+	w.s.m.Unlock()
+
+	return newIterator(t).restart(&Item{k: k})
+}
+
+func (w *Writer) Close() error {
+	w.s.availableWriters <- true
+	w.s = nil
+
+	return nil
+}
+
+func (w *Writer) Set(k, v []byte) (err error) {
+	w.s.m.Lock()
+	w.s.t = w.s.t.Upsert(&Item{k: k, v: v}, rand.Int())
+	w.s.m.Unlock()
+
+	return nil
+}
+
+func (w *Writer) Delete(k []byte) (err error) {
+	w.s.m.Lock()
+	w.s.t = w.s.t.Delete(&Item{k: k})
+	w.s.m.Unlock()
+
+	return nil
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	return store.NewEmulatedBatch(w, w.s.mo)
+}

vendor/github.com/blevesearch/bleve/index/store/inmem/iterator.go

@@ -0,0 +1,70 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package inmem
+
+import (
+	"github.com/ryszard/goskiplist/skiplist"
+)
+
+type Iterator struct {
+	store    *Store
+	iterator skiplist.Iterator
+	valid    bool
+}
+
+func newIterator(store *Store) *Iterator {
+	rv := Iterator{
+		store:    store,
+		iterator: store.list.Iterator(),
+	}
+	return &rv
+}
+
+func (i *Iterator) SeekFirst() {
+	i.Seek([]byte{0})
+}
+
+func (i *Iterator) Seek(k []byte) {
+	i.valid = i.iterator.Seek(string(k))
+}
+
+func (i *Iterator) Next() {
+	i.valid = i.iterator.Next()
+}
+
+func (i *Iterator) Current() ([]byte, []byte, bool) {
+	if i.valid {
+		return []byte(i.Key()), []byte(i.Value()), true
+	}
+	return nil, nil, false
+}
+
+func (i *Iterator) Key() []byte {
+	if i.valid {
+		return []byte(i.iterator.Key().(string))
+	}
+	return nil
+}
+
+func (i *Iterator) Value() []byte {
+	if i.valid {
+		return []byte(i.iterator.Value().(string))
+	}
+	return nil
+}
+
+func (i *Iterator) Valid() bool {
+	return i.valid
+}
+
+func (i *Iterator) Close() error {
+	i.iterator.Close()
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/inmem/reader.go

@@ -0,0 +1,40 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package inmem
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+)
+
+type Reader struct {
+	store *Store
+}
+
+func newReader(store *Store) (*Reader, error) {
+	return &Reader{
+		store: store,
+	}, nil
+}
+
+func (r *Reader) BytesSafeAfterClose() bool {
+	return false
+}
+
+func (r *Reader) Get(key []byte) ([]byte, error) {
+	return r.store.get(key)
+}
+
+func (r *Reader) Iterator(key []byte) store.KVIterator {
+	return r.store.iterator(key)
+}
+
+func (r *Reader) Close() error {
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/inmem/store.go

@@ -0,0 +1,106 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package inmem
+
+import (
+	"sync"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/registry"
+	"github.com/ryszard/goskiplist/skiplist"
+)
+
+const Name = "mem"
+
+type Store struct {
+	list   *skiplist.SkipList
+	writer sync.Mutex
+	mo     store.MergeOperator
+}
+
+func New() (*Store, error) {
+	rv := Store{
+		list: skiplist.NewStringMap(),
+	}
+
+	return &rv, nil
+}
+
+func MustOpen() *Store {
+	rv := Store{
+		list: skiplist.NewStringMap(),
+	}
+
+	return &rv
+}
+
+func (i *Store) Open() error {
+	return nil
+}
+
+func (i *Store) SetMergeOperator(mo store.MergeOperator) {
+	i.mo = mo
+}
+
+func (i *Store) get(key []byte) ([]byte, error) {
+	val, ok := i.list.Get(string(key))
+	if ok {
+		return []byte(val.(string)), nil
+	}
+	return nil, nil
+}
+
+func (i *Store) set(key, val []byte) error {
+	i.writer.Lock()
+	defer i.writer.Unlock()
+	return i.setlocked(key, val)
+}
+
+func (i *Store) setlocked(key, val []byte) error {
+	i.list.Set(string(key), string(val))
+	return nil
+}
+
+func (i *Store) delete(key []byte) error {
+	i.writer.Lock()
+	defer i.writer.Unlock()
+	return i.deletelocked(key)
+}
+
+func (i *Store) deletelocked(key []byte) error {
+	i.list.Delete(string(key))
+	return nil
+}
+
+func (i *Store) Close() error {
+	return nil
+}
+
+func (i *Store) iterator(key []byte) store.KVIterator {
+	rv := newIterator(i)
+	rv.Seek(key)
+	return rv
+}
+
+func (i *Store) Reader() (store.KVReader, error) {
+	return newReader(i)
+}
+
+func (i *Store) Writer() (store.KVWriter, error) {
+	return newWriter(i)
+}
+
+func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
+	return New()
+}
+
+func init() {
+	registry.RegisterKVStore(Name, StoreConstructor)
+}

vendor/github.com/blevesearch/bleve/index/store/inmem/store_test.go

@@ -0,0 +1,254 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package inmem
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+)
+
+func TestStore(t *testing.T) {
+	s, err := New()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := s.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	CommonTestKVStore(t, s)
+}
+
+func CommonTestKVStore(t *testing.T, s store.KVStore) {
+
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Set([]byte("z"), []byte("val-z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Delete([]byte("z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	batch := writer.NewBatch()
+	batch.Set([]byte("b"), []byte("val-b"))
+	batch.Set([]byte("c"), []byte("val-c"))
+	batch.Set([]byte("d"), []byte("val-d"))
+	batch.Set([]byte("e"), []byte("val-e"))
+	batch.Set([]byte("f"), []byte("val-f"))
+	batch.Set([]byte("g"), []byte("val-g"))
+	batch.Set([]byte("h"), []byte("val-h"))
+	batch.Set([]byte("i"), []byte("val-i"))
+	batch.Set([]byte("j"), []byte("val-j"))
+
+	err = batch.Execute()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	it := reader.Iterator([]byte("b"))
+	key, val, valid := it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "b" {
+		t.Fatalf("expected key b, got %s", key)
+	}
+	if string(val) != "val-b" {
+		t.Fatalf("expected value val-b, got %s", val)
+	}
+
+	it.Next()
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "c" {
+		t.Fatalf("expected key c, got %s", key)
+	}
+	if string(val) != "val-c" {
+		t.Fatalf("expected value val-c, got %s", val)
+	}
+
+	it.Seek([]byte("i"))
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "i" {
+		t.Fatalf("expected key i, got %s", key)
+	}
+	if string(val) != "val-i" {
+		t.Fatalf("expected value val-i, got %s", val)
+	}
+
+	err = it.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
+	// insert a kv pair
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create an isolated reader
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	// verify that we see the value already inserted
+	val, err := reader.Get([]byte("a"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-a")) {
+		t.Errorf("expected val-a, got nil")
+	}
+
+	// verify that an iterator sees it
+	count := 0
+	it := reader.Iterator([]byte{0})
+	defer func() {
+		err := it.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it.Valid() {
+		it.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+	// add something after the reader was created
+	writer, err = s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("b"), []byte("val-b"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ensure that a newer reader sees it
+	newReader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := newReader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	val, err = newReader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-b")) {
+		t.Errorf("expected val-b, got nil")
+	}
+
+	// ensure that the director iterator sees it
+	count = 0
+	it = newReader.Iterator([]byte{0})
+	defer func() {
+		err := it.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it.Valid() {
+		it.Next()
+		count++
+	}
+	if count != 2 {
+		t.Errorf("expected iterator to see 2, saw %d", count)
+	}
+
+	// but that the isolated reader does not
+	val, err = reader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if val != nil {
+		t.Errorf("expected nil, got %v", val)
+	}
+
+	// and ensure that the iterator on the isolated reader also does not
+	count = 0
+	it = reader.Iterator([]byte{0})
+	defer func() {
+		err := it.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it.Valid() {
+		it.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+}

vendor/github.com/blevesearch/bleve/index/store/inmem/writer.go

@@ -0,0 +1,57 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package inmem
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+)
+
+type Writer struct {
+	store *Store
+}
+
+func newWriter(store *Store) (*Writer, error) {
+	store.writer.Lock()
+	return &Writer{
+		store: store,
+	}, nil
+}
+
+func (w *Writer) BytesSafeAfterClose() bool {
+	return false
+}
+
+func (w *Writer) Set(key, val []byte) error {
+	return w.store.setlocked(key, val)
+}
+
+func (w *Writer) Delete(key []byte) error {
+	return w.store.deletelocked(key)
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	return store.NewEmulatedBatch(w, w.store.mo)
+}
+
+func (w *Writer) Close() error {
+	w.store.writer.Unlock()
+	return nil
+}
+
+// these two methods can safely read using the regular
+// methods without a read transaction, because we know
+// that no one else is writing but us
+func (w *Writer) Get(key []byte) ([]byte, error) {
+	return w.store.get(key)
+}
+
+func (w *Writer) Iterator(key []byte) store.KVIterator {
+	return w.store.iterator(key)
+}

vendor/github.com/blevesearch/bleve/index/store/kvstore.go

@@ -0,0 +1,53 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package store
+
+type KVBatch interface {
+	Set(key, val []byte)
+	Delete(key []byte)
+	Merge(key, val []byte)
+	Execute() error
+	Close() error
+}
+
+type KVIterator interface {
+	SeekFirst()
+	Seek([]byte)
+	Next()
+
+	Current() ([]byte, []byte, bool)
+	Key() []byte
+	Value() []byte
+	Valid() bool
+
+	Close() error
+}
+
+type KVStore interface {
+	Open() error
+	SetMergeOperator(MergeOperator)
+	Writer() (KVWriter, error)
+	Reader() (KVReader, error)
+	Close() error
+}
+
+type KVWriter interface {
+	KVReader
+	Set(key, val []byte) error
+	Delete(key []byte) error
+	NewBatch() KVBatch
+}
+
+type KVReader interface {
+	BytesSafeAfterClose() bool
+	Get(key []byte) ([]byte, error)
+	Iterator(key []byte) KVIterator
+	Close() error
+}

vendor/github.com/blevesearch/bleve/index/store/leveldb/batch.go

@@ -0,0 +1,55 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build leveldb full
+
+package leveldb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/jmhodges/levigo"
+)
+
+type Batch struct {
+	w     *Writer
+	merge *store.EmulatedMerge
+	batch *levigo.WriteBatch
+}
+
+func (b *Batch) Set(key, val []byte) {
+	b.batch.Put(key, val)
+}
+
+func (b *Batch) Delete(key []byte) {
+	b.batch.Delete(key)
+}
+
+func (b *Batch) Merge(key, val []byte) {
+	b.merge.Merge(key, val)
+}
+
+func (b *Batch) Execute() error {
+	// first process merges
+	ops, err := b.merge.ExecuteDeferred(b.w)
+	if err != nil {
+		return err
+	}
+	for _, op := range ops {
+		b.batch.Put(op.K, op.V)
+	}
+
+	wopts := defaultWriteOptions()
+	defer wopts.Close()
+	err = b.w.store.db.Write(wopts, b.batch)
+	return err
+}
+
+func (b *Batch) Close() error {
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/leveldb/iterator.go

@@ -0,0 +1,78 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build leveldb full
+
+package leveldb
+
+import (
+	"github.com/jmhodges/levigo"
+)
+
+type Iterator struct {
+	store    *Store
+	iterator *levigo.Iterator
+}
+
+func newIterator(store *Store) *Iterator {
+	ropts := defaultReadOptions()
+	rv := Iterator{
+		store:    store,
+		iterator: store.db.NewIterator(ropts),
+	}
+	ropts.Close()
+	return &rv
+}
+
+func newIteratorWithSnapshot(store *Store, snapshot *levigo.Snapshot) *Iterator {
+	options := defaultReadOptions()
+	options.SetSnapshot(snapshot)
+	rv := Iterator{
+		store:    store,
+		iterator: store.db.NewIterator(options),
+	}
+	options.Close()
+	return &rv
+}
+
+func (ldi *Iterator) SeekFirst() {
+	ldi.iterator.SeekToFirst()
+}
+
+func (ldi *Iterator) Seek(key []byte) {
+	ldi.iterator.Seek(key)
+}
+
+func (ldi *Iterator) Next() {
+	ldi.iterator.Next()
+}
+
+func (ldi *Iterator) Current() ([]byte, []byte, bool) {
+	if ldi.Valid() {
+		return ldi.Key(), ldi.Value(), true
+	}
+	return nil, nil, false
+}
+
+func (ldi *Iterator) Key() []byte {
+	return ldi.iterator.Key()
+}
+
+func (ldi *Iterator) Value() []byte {
+	return ldi.iterator.Value()
+}
+
+func (ldi *Iterator) Valid() bool {
+	return ldi.iterator.Valid()
+}
+
+func (ldi *Iterator) Close() error {
+	ldi.iterator.Close()
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/leveldb/reader.go

@@ -0,0 +1,48 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build leveldb full
+
+package leveldb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/jmhodges/levigo"
+)
+
+type Reader struct {
+	store    *Store
+	snapshot *levigo.Snapshot
+}
+
+func newReader(store *Store) (*Reader, error) {
+	return &Reader{
+		store:    store,
+		snapshot: store.db.NewSnapshot(),
+	}, nil
+}
+
+func (r *Reader) BytesSafeAfterClose() bool {
+	return true
+}
+
+func (r *Reader) Get(key []byte) ([]byte, error) {
+	return r.store.getWithSnapshot(key, r.snapshot)
+}
+
+func (r *Reader) Iterator(key []byte) store.KVIterator {
+	rv := newIteratorWithSnapshot(r.store, r.snapshot)
+	rv.Seek(key)
+	return rv
+}
+
+func (r *Reader) Close() error {
+	r.store.db.ReleaseSnapshot(r.snapshot)
+	return nil
+}

vendor/github.com/blevesearch/bleve/index/store/leveldb/store.go

@@ -0,0 +1,174 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build leveldb full
+
+package leveldb
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/registry"
+	"github.com/jmhodges/levigo"
+)
+
+const Name = "leveldb"
+
+type Store struct {
+	path   string
+	opts   *levigo.Options
+	db     *levigo.DB
+	writer sync.Mutex
+	mo     store.MergeOperator
+}
+
+func New(path string, config map[string]interface{}) (*Store, error) {
+	rv := Store{
+		path: path,
+		opts: levigo.NewOptions(),
+	}
+
+	_, err := applyConfig(rv.opts, config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &rv, nil
+}
+
+func (ldbs *Store) Open() error {
+	var err error
+	ldbs.db, err = levigo.Open(ldbs.path, ldbs.opts)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (ldbs *Store) SetMergeOperator(mo store.MergeOperator) {
+	ldbs.mo = mo
+}
+
+func (ldbs *Store) get(key []byte) ([]byte, error) {
+	options := defaultReadOptions()
+	b, err := ldbs.db.Get(options, key)
+	options.Close()
+	return b, err
+}
+
+func (ldbs *Store) getWithSnapshot(key []byte, snapshot *levigo.Snapshot) ([]byte, error) {
+	options := defaultReadOptions()
+	options.SetSnapshot(snapshot)
+	b, err := ldbs.db.Get(options, key)
+	options.Close()
+	return b, err
+}
+
+func (ldbs *Store) set(key, val []byte) error {
+	ldbs.writer.Lock()
+	defer ldbs.writer.Unlock()
+	return ldbs.setlocked(key, val)
+}
+
+func (ldbs *Store) setlocked(key, val []byte) error {
+	options := defaultWriteOptions()
+	err := ldbs.db.Put(options, key, val)
+	options.Close()
+	return err
+}
+
+func (ldbs *Store) delete(key []byte) error {
+	ldbs.writer.Lock()
+	defer ldbs.writer.Unlock()
+	return ldbs.deletelocked(key)
+}
+
+func (ldbs *Store) deletelocked(key []byte) error {
+	options := defaultWriteOptions()
+	err := ldbs.db.Delete(options, key)
+	options.Close()
+	return err
+}
+
+func (ldbs *Store) Close() error {
+	ldbs.db.Close()
+	ldbs.opts.Close()
+	return nil
+}
+
+func (ldbs *Store) iterator(key []byte) store.KVIterator {
+	rv := newIterator(ldbs)
+	rv.Seek(key)
+	return rv
+}
+
+func (ldbs *Store) Reader() (store.KVReader, error) {
+	return newReader(ldbs)
+}
+
+func (ldbs *Store) Writer() (store.KVWriter, error) {
+	return newWriter(ldbs)
+}
+
+func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
+	path, ok := config["path"].(string)
+	if !ok {
+		return nil, fmt.Errorf("must specify path")
+	}
+	return New(path, config)
+}
+
+func init() {
+	registry.RegisterKVStore(Name, StoreConstructor)
+}
+
+func applyConfig(o *levigo.Options, config map[string]interface{}) (
+	*levigo.Options, error) {
+
+	cim, ok := config["create_if_missing"].(bool)
+	if ok {
+		o.SetCreateIfMissing(cim)
+	}
+
+	eie, ok := config["error_if_exists"].(bool)
+	if ok {
+		o.SetErrorIfExists(eie)
+	}
+
+	wbs, ok := config["write_buffer_size"].(float64)
+	if ok {
+		o.SetWriteBufferSize(int(wbs))
+	}
+
+	bs, ok := config["block_size"].(float64)
+	if ok {
+		o.SetBlockSize(int(bs))
+	}
+
+	bri, ok := config["block_restart_interval"].(float64)
+	if ok {
+		o.SetBlockRestartInterval(int(bri))
+	}
+
+	lcc, ok := config["lru_cache_capacity"].(float64)
+	if ok {
+		lruCache := levigo.NewLRUCache(int(lcc))
+		o.SetCache(lruCache)
+	}
+
+	bfbpk, ok := config["bloom_filter_bits_per_key"].(float64)
+	if ok {
+		bf := levigo.NewBloomFilter(int(bfbpk))
+		o.SetFilterPolicy(bf)
+	}
+
+	return o, nil
+}

vendor/github.com/blevesearch/bleve/index/store/leveldb/store_test.go

@@ -0,0 +1,298 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build leveldb full
+
+package leveldb
+
+import (
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+)
+
+var leveldbTestOptions = map[string]interface{}{
+	"create_if_missing": true,
+}
+
+func TestLevelDBStore(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	s, err := New("test", leveldbTestOptions)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := s.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	CommonTestKVStore(t, s)
+}
+
+func TestReaderIsolation(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	s, err := New("test", leveldbTestOptions)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = s.Open()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() {
+		err := s.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	CommonTestReaderIsolation(t, s)
+}
+
+func CommonTestKVStore(t *testing.T, s store.KVStore) {
+
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Set([]byte("z"), []byte("val-z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Delete([]byte("z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	batch := writer.NewBatch()
+	batch.Set([]byte("b"), []byte("val-b"))
+	batch.Set([]byte("c"), []byte("val-c"))
+	batch.Set([]byte("d"), []byte("val-d"))
+	batch.Set([]byte("e"), []byte("val-e"))
+	batch.Set([]byte("f"), []byte("val-f"))
+	batch.Set([]byte("g"), []byte("val-g"))
+	batch.Set([]byte("h"), []byte("val-h"))
+	batch.Set([]byte("i"), []byte("val-i"))
+	batch.Set([]byte("j"), []byte("val-j"))
+
+	err = batch.Execute()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	it := reader.Iterator([]byte("b"))
+	key, val, valid := it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "b" {
+		t.Fatalf("expected key b, got %s", key)
+	}
+	if string(val) != "val-b" {
+		t.Fatalf("expected value val-b, got %s", val)
+	}
+
+	it.Next()
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "c" {
+		t.Fatalf("expected key c, got %s", key)
+	}
+	if string(val) != "val-c" {
+		t.Fatalf("expected value val-c, got %s", val)
+	}
+
+	it.Seek([]byte("i"))
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "i" {
+		t.Fatalf("expected key i, got %s", key)
+	}
+	if string(val) != "val-i" {
+		t.Fatalf("expected value val-i, got %s", val)
+	}
+
+	err = it.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
+	// insert a kv pair
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create an isolated reader
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	// verify that we see the value already inserted
+	val, err := reader.Get([]byte("a"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-a")) {
+		t.Errorf("expected val-a, got nil")
+	}
+
+	// verify that an iterator sees it
+	count := 0
+	it := reader.Iterator([]byte{0})
+	defer func() {
+		err := it.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it.Valid() {
+		it.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+	// add something after the reader was created
+	writer, err = s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("b"), []byte("val-b"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ensure that a newer reader sees it
+	newReader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := newReader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	val, err = newReader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-b")) {
+		t.Errorf("expected val-b, got nil")
+	}
+
+	// ensure that the director iterator sees it
+	count = 0
+	it2 := newReader.Iterator([]byte{0})
+	defer func() {
+		err := it2.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it2.Valid() {
+		it2.Next()
+		count++
+	}
+	if count != 2 {
+		t.Errorf("expected iterator to see 2, saw %d", count)
+	}
+
+	// but that the isolated reader does not
+	val, err = reader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if val != nil {
+		t.Errorf("expected nil, got %v", val)
+	}
+
+	// and ensure that the iterator on the isolated reader also does not
+	count = 0
+	it3 := reader.Iterator([]byte{0})
+	defer func() {
+		err := it3.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it3.Valid() {
+		it3.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+}

vendor/github.com/blevesearch/bleve/index/store/leveldb/util.go

@@ -0,0 +1,28 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build leveldb full
+
+package leveldb
+
+import (
+	"github.com/jmhodges/levigo"
+)
+
+func defaultWriteOptions() *levigo.WriteOptions {
+	wo := levigo.NewWriteOptions()
+	// request fsync on write for safety
+	wo.SetSync(true)
+	return wo
+}
+
+func defaultReadOptions() *levigo.ReadOptions {
+	ro := levigo.NewReadOptions()
+	return ro
+}

vendor/github.com/blevesearch/bleve/index/store/leveldb/writer.go

@@ -0,0 +1,65 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build leveldb full
+
+package leveldb
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/jmhodges/levigo"
+)
+
+type Writer struct {
+	store *Store
+}
+
+func newWriter(store *Store) (*Writer, error) {
+	store.writer.Lock()
+	return &Writer{
+		store: store,
+	}, nil
+}
+
+func (w *Writer) BytesSafeAfterClose() bool {
+	return true
+}
+
+func (w *Writer) Set(key, val []byte) error {
+	return w.store.setlocked(key, val)
+}
+
+func (w *Writer) Delete(key []byte) error {
+	return w.store.deletelocked(key)
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	rv := Batch{
+		w:     w,
+		merge: store.NewEmulatedMerge(w.store.mo),
+		batch: levigo.NewWriteBatch(),
+	}
+	return &rv
+}
+
+func (w *Writer) Close() error {
+	w.store.writer.Unlock()
+	return nil
+}
+
+// these two methods can safely read using the regular
+// methods without a read transaction, because we know
+// that no one else is writing but us
+func (w *Writer) Get(key []byte) ([]byte, error) {
+	return w.store.get(key)
+}
+
+func (w *Writer) Iterator(key []byte) store.KVIterator {
+	return w.store.iterator(key)
+}

vendor/github.com/blevesearch/bleve/index/store/merge.go

@@ -0,0 +1,120 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package store
+
+import (
+	"fmt"
+)
+
+// At the moment this happens to be the same interface as described by
+// RocksDB, but this may not always be the case.
+
+type MergeOperator interface {
+
+	// FullMerge the full sequence of operands on top of the existingValue
+	// if no value currently exists, existingValue is nil
+	// return the merged value, and success/failure
+	FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool)
+
+	// Partially merge these two operands.
+	// If partial merge cannot be done, return nil,false, which will defer
+	// all processing until the FullMerge is done.
+	PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool)
+
+	// Name returns an identifier for the operator
+	Name() string
+}
+
+// EmulatedMergeSingle removes some duplicated code across
+// KV stores which do not support merge operations
+// on their own.  It is up to the caller to ensure
+// that an appropriate lock has been acquired in
+// order for this behavior to be valid
+func EmulatedMergeSingle(writer KVWriter, mo MergeOperator, key []byte, operand []byte) error {
+	existingValue, err := writer.Get(key)
+	if err != nil {
+		return err
+	}
+	newValue, ok := mo.FullMerge(key, existingValue, [][]byte{operand})
+	if !ok {
+		return fmt.Errorf("merge operator returned failure")
+	}
+	err = writer.Set(key, newValue)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+type EmulatedMerge struct {
+	merges map[string][][]byte
+	mo     MergeOperator
+}
+
+func NewEmulatedMerge(mo MergeOperator) *EmulatedMerge {
+	return &EmulatedMerge{
+		merges: make(map[string][][]byte),
+		mo:     mo,
+	}
+}
+
+func (m *EmulatedMerge) Merge(key, val []byte) {
+	ops, ok := m.merges[string(key)]
+	if ok && len(ops) > 0 {
+		last := ops[len(ops)-1]
+		mergedVal, partialMergeOk := m.mo.PartialMerge(key, last, val)
+		if partialMergeOk {
+			// replace last entry with the result of the merge
+			ops[len(ops)-1] = mergedVal
+		} else {
+			// could not partial merge, append this to the end
+			ops = append(ops, val)
+		}
+	} else {
+		ops = [][]byte{val}
+	}
+	m.merges[string(key)] = ops
+}
+
+func (m *EmulatedMerge) Execute(w KVWriter) error {
+	for k, mergeOps := range m.merges {
+		kb := []byte(k)
+		existingVal, err := w.Get(kb)
+		if err != nil {
+			return err
+		}
+		mergedVal, fullMergeOk := m.mo.FullMerge(kb, existingVal, mergeOps)
+		if !fullMergeOk {
+			return fmt.Errorf("merge operator returned failure")
+		}
+		err = w.Set(kb, mergedVal)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (m *EmulatedMerge) ExecuteDeferred(w KVWriter) ([]*op, error) {
+	rv := make([]*op, 0, 1000)
+	for k, mergeOps := range m.merges {
+		kb := []byte(k)
+		existingVal, err := w.Get(kb)
+		if err != nil {
+			return nil, err
+		}
+		mergedVal, fullMergeOk := m.mo.FullMerge(kb, existingVal, mergeOps)
+		if !fullMergeOk {
+			return nil, fmt.Errorf("merge operator returned failure")
+		}
+		rv = append(rv, &op{kb, mergedVal})
+	}
+	return rv, nil
+}

vendor/github.com/blevesearch/bleve/index/store/metrics/metrics.go

@@ -0,0 +1,476 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the
+//  License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing,
+//  software distributed under the License is distributed on an "AS
+//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+//  express or implied. See the License for the specific language
+//  governing permissions and limitations under the License.
+
+// +build debug
+
+// Package metrics provides a bleve.store.KVStore implementation that
+// wraps another, real KVStore implementation, and uses go-metrics to
+// track runtime performance metrics.
+package metrics
+
+import (
+	"container/list"
+	"encoding/json"
+	"fmt"
+	"io"
+	"sync"
+	"time"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/registry"
+
+	"github.com/rcrowley/go-metrics"
+)
+
+const Name = "metrics"
+const MaxErrors = 100
+
+func init() {
+	registry.RegisterKVStore(Name, StoreConstructor)
+}
+
+func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
+	name, ok := config["kvStoreName_actual"].(string)
+	if !ok || name == "" {
+		return nil, fmt.Errorf("metrics: missing kvStoreName_actual,"+
+			" config: %#v", config)
+	}
+
+	if name == Name {
+		return nil, fmt.Errorf("metrics: circular kvStoreName_actual")
+	}
+
+	ctr := registry.KVStoreConstructorByName(name)
+	if ctr == nil {
+		return nil, fmt.Errorf("metrics: no kv store constructor,"+
+			" kvStoreName_actual: %s", name)
+	}
+
+	kvs, err := ctr(config)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewBleveMetricsStore(kvs), nil
+}
+
+func NewBleveMetricsStore(o store.KVStore) *Store {
+	return &Store{
+		o: o,
+
+		TimerReaderGet:         metrics.NewTimer(),
+		TimerReaderIterator:    metrics.NewTimer(),
+		TimerWriterGet:         metrics.NewTimer(),
+		TimerWriterIterator:    metrics.NewTimer(),
+		TimerWriterSet:         metrics.NewTimer(),
+		TimerWriterDelete:      metrics.NewTimer(),
+		TimerIteratorSeekFirst: metrics.NewTimer(),
+		TimerIteratorSeek:      metrics.NewTimer(),
+		TimerIteratorNext:      metrics.NewTimer(),
+		TimerBatchMerge:        metrics.NewTimer(),
+		TimerBatchExecute:      metrics.NewTimer(),
+
+		errors: list.New(),
+	}
+}
+
+// The following structs are wrappers around "real" bleve kvstore
+// implementations.
+
+type Store struct {
+	o store.KVStore
+
+	TimerReaderGet         metrics.Timer
+	TimerReaderIterator    metrics.Timer
+	TimerWriterGet         metrics.Timer
+	TimerWriterIterator    metrics.Timer
+	TimerWriterSet         metrics.Timer
+	TimerWriterDelete      metrics.Timer
+	TimerIteratorSeekFirst metrics.Timer
+	TimerIteratorSeek      metrics.Timer
+	TimerIteratorNext      metrics.Timer
+	TimerBatchMerge        metrics.Timer
+	TimerBatchExecute      metrics.Timer
+
+	m      sync.Mutex // Protects the fields that follow.
+	errors *list.List // Capped list of StoreError's.
+}
+
+type StoreError struct {
+	Time string
+	Op   string
+	Err  string
+	Key  string
+}
+
+type Reader struct {
+	s *Store
+	o store.KVReader
+}
+
+type Writer struct {
+	s *Store
+	o store.KVWriter
+}
+
+type Iterator struct {
+	s *Store
+	o store.KVIterator
+}
+
+type Batch struct {
+	s *Store
+	o store.KVBatch
+}
+
+func (s *Store) Close() error {
+	return s.o.Close()
+}
+
+func (s *Store) Reader() (store.KVReader, error) {
+	o, err := s.o.Reader()
+	if err != nil {
+		s.AddError("Reader", err, nil)
+		return nil, err
+	}
+	return &Reader{s: s, o: o}, nil
+}
+
+func (s *Store) Writer() (store.KVWriter, error) {
+	o, err := s.o.Writer()
+	if err != nil {
+		s.AddError("Writer", err, nil)
+		return nil, err
+	}
+	return &Writer{s: s, o: o}, nil
+}
+
+func (s *Store) Actual() store.KVStore {
+	return s.o
+}
+
+func (w *Reader) BytesSafeAfterClose() bool {
+	return w.o.BytesSafeAfterClose()
+}
+
+func (w *Reader) Get(key []byte) (v []byte, err error) {
+	w.s.TimerReaderGet.Time(func() {
+		v, err = w.o.Get(key)
+		if err != nil {
+			w.s.AddError("Reader.Get", err, key)
+		}
+	})
+	return
+}
+
+func (w *Reader) Iterator(key []byte) (i store.KVIterator) {
+	w.s.TimerReaderIterator.Time(func() {
+		i = &Iterator{s: w.s, o: w.o.Iterator(key)}
+	})
+	return
+}
+
+func (w *Reader) Close() error {
+	err := w.o.Close()
+	if err != nil {
+		w.s.AddError("Reader.Close", err, nil)
+	}
+	return err
+}
+
+func (w *Writer) BytesSafeAfterClose() bool {
+	return w.o.BytesSafeAfterClose()
+}
+
+func (w *Writer) Get(key []byte) (v []byte, err error) {
+	w.s.TimerWriterGet.Time(func() {
+		v, err = w.o.Get(key)
+		if err != nil {
+			w.s.AddError("Writer.Get", err, key)
+		}
+	})
+	return
+}
+
+func (w *Writer) Iterator(key []byte) (i store.KVIterator) {
+	w.s.TimerWriterIterator.Time(func() {
+		i = &Iterator{s: w.s, o: w.o.Iterator(key)}
+	})
+	return
+}
+
+func (w *Writer) Close() error {
+	err := w.o.Close()
+	if err != nil {
+		w.s.AddError("Writer.Close", err, nil)
+	}
+	return err
+}
+
+func (w *Writer) Set(key, val []byte) (err error) {
+	w.s.TimerWriterSet.Time(func() {
+		err = w.o.Set(key, val)
+		if err != nil {
+			w.s.AddError("Writer.Set", err, key)
+		}
+	})
+	return
+}
+
+func (w *Writer) Delete(key []byte) (err error) {
+	w.s.TimerWriterDelete.Time(func() {
+		err = w.o.Delete(key)
+		if err != nil {
+			w.s.AddError("Writer.Delete", err, key)
+		}
+	})
+	return
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	return &Batch{s: w.s, o: w.o.NewBatch()}
+}
+
+func (w *Iterator) SeekFirst() {
+	w.s.TimerIteratorSeekFirst.Time(func() {
+		w.o.SeekFirst()
+	})
+}
+
+func (w *Iterator) Seek(x []byte) {
+	w.s.TimerIteratorSeek.Time(func() {
+		w.o.Seek(x)
+	})
+}
+
+func (w *Iterator) Next() {
+	w.s.TimerIteratorNext.Time(func() {
+		w.o.Next()
+	})
+}
+
+func (w *Iterator) Current() ([]byte, []byte, bool) {
+	return w.o.Current()
+}
+
+func (w *Iterator) Key() []byte {
+	return w.o.Key()
+}
+
+func (w *Iterator) Value() []byte {
+	return w.o.Value()
+}
+
+func (w *Iterator) Valid() bool {
+	return w.o.Valid()
+}
+
+func (w *Iterator) Close() error {
+	err := w.o.Close()
+	if err != nil {
+		w.s.AddError("Iterator.Close", err, nil)
+	}
+	return err
+}
+
+func (w *Batch) Set(key, val []byte) {
+	w.o.Set(key, val)
+}
+
+func (w *Batch) Delete(key []byte) {
+	w.o.Delete(key)
+}
+
+func (w *Batch) Merge(key []byte, oper store.AssociativeMerge) {
+	w.s.TimerBatchMerge.Time(func() {
+		w.o.Merge(key, oper)
+	})
+}
+
+func (w *Batch) Execute() (err error) {
+	w.s.TimerBatchExecute.Time(func() {
+		err = w.o.Execute()
+		if err != nil {
+			w.s.AddError("Batch.Execute", err, nil)
+		}
+	})
+	return
+}
+
+func (w *Batch) Close() error {
+	err := w.o.Close()
+	if err != nil {
+		w.s.AddError("Batch.Close", err, nil)
+	}
+	return err
+}
+
+// --------------------------------------------------------
+
+func (s *Store) AddError(op string, err error, key []byte) {
+	e := &StoreError{
+		Time: time.Now().Format(time.RFC3339Nano),
+		Op:   op,
+		Err:  fmt.Sprintf("%v", err),
+		Key:  string(key),
+	}
+
+	s.m.Lock()
+	for s.errors.Len() >= MaxErrors {
+		s.errors.Remove(s.errors.Front())
+	}
+	s.errors.PushBack(e)
+	s.m.Unlock()
+}
+
+// --------------------------------------------------------
+
+func (s *Store) WriteJSON(w io.Writer) {
+	w.Write([]byte(`{"TimerReaderGet":`))
+	WriteTimerJSON(w, s.TimerReaderGet)
+	w.Write([]byte(`,"TimerReaderIterator":`))
+	WriteTimerJSON(w, s.TimerReaderIterator)
+	w.Write([]byte(`,"TimerWriterGet":`))
+	WriteTimerJSON(w, s.TimerWriterGet)
+	w.Write([]byte(`,"TimerWriterIterator":`))
+	WriteTimerJSON(w, s.TimerWriterIterator)
+	w.Write([]byte(`,"TimerWriterSet":`))
+	WriteTimerJSON(w, s.TimerWriterSet)
+	w.Write([]byte(`,"TimerWriterDelete":`))
+	WriteTimerJSON(w, s.TimerWriterDelete)
+	w.Write([]byte(`,"TimerIteratorSeekFirst":`))
+	WriteTimerJSON(w, s.TimerIteratorSeekFirst)
+	w.Write([]byte(`,"TimerIteratorSeek":`))
+	WriteTimerJSON(w, s.TimerIteratorSeek)
+	w.Write([]byte(`,"TimerIteratorNext":`))
+	WriteTimerJSON(w, s.TimerIteratorNext)
+	w.Write([]byte(`,"TimerBatchMerge":`))
+	WriteTimerJSON(w, s.TimerBatchMerge)
+	w.Write([]byte(`,"TimerBatchExecute":`))
+	WriteTimerJSON(w, s.TimerBatchExecute)
+
+	w.Write([]byte(`,"Errors":[`))
+	s.m.Lock()
+	e := s.errors.Front()
+	i := 0
+	for e != nil {
+		se, ok := e.Value.(*StoreError)
+		if ok && se != nil {
+			if i > 0 {
+				w.Write([]byte(","))
+			}
+			buf, err := json.Marshal(se)
+			if err == nil {
+				w.Write(buf)
+			}
+		}
+		e = e.Next()
+		i = i + 1
+	}
+	s.m.Unlock()
+	w.Write([]byte(`]`))
+
+	w.Write([]byte(`}`))
+}
+
+func (s *Store) WriteCSVHeader(w io.Writer) {
+	WriteTimerCSVHeader(w, "TimerReaderGet")
+	WriteTimerCSVHeader(w, "TimerReaderIterator")
+	WriteTimerCSVHeader(w, "TimerWriterGet")
+	WriteTimerCSVHeader(w, "TimerWriterIterator")
+	WriteTimerCSVHeader(w, "TimerWriterSet")
+	WriteTimerCSVHeader(w, "TimerWriterDelete")
+	WriteTimerCSVHeader(w, "TimerIteratorSeekFirst")
+	WriteTimerCSVHeader(w, "TimerIteratorSeek")
+	WriteTimerCSVHeader(w, "TimerIteratorNext")
+	WriteTimerCSVHeader(w, "TimerBatchMerge")
+	WriteTimerCSVHeader(w, "TimerBatchExecute")
+}
+
+func (s *Store) WriteCSV(w io.Writer) {
+	WriteTimerCSV(w, s.TimerReaderGet)
+	WriteTimerCSV(w, s.TimerReaderIterator)
+	WriteTimerCSV(w, s.TimerWriterGet)
+	WriteTimerCSV(w, s.TimerWriterIterator)
+	WriteTimerCSV(w, s.TimerWriterSet)
+	WriteTimerCSV(w, s.TimerWriterDelete)
+	WriteTimerCSV(w, s.TimerIteratorSeekFirst)
+	WriteTimerCSV(w, s.TimerIteratorSeek)
+	WriteTimerCSV(w, s.TimerIteratorNext)
+	WriteTimerCSV(w, s.TimerBatchMerge)
+	WriteTimerCSV(w, s.TimerBatchExecute)
+}
+
+// --------------------------------------------------------
+
+// NOTE: This is copy & pasted from cbft as otherwise there
+// would be an import cycle.
+
+var timerPercentiles = []float64{0.5, 0.75, 0.95, 0.99, 0.999}
+
+func WriteTimerJSON(w io.Writer, timer metrics.Timer) {
+	t := timer.Snapshot()
+	p := t.Percentiles(timerPercentiles)
+
+	fmt.Fprintf(w, `{"count":%9d,`, t.Count())
+	fmt.Fprintf(w, `"min":%9d,`, t.Min())
+	fmt.Fprintf(w, `"max":%9d,`, t.Max())
+	fmt.Fprintf(w, `"mean":%12.2f,`, t.Mean())
+	fmt.Fprintf(w, `"stddev":%12.2f,`, t.StdDev())
+	fmt.Fprintf(w, `"percentiles":{`)
+	fmt.Fprintf(w, `"median":%12.2f,`, p[0])
+	fmt.Fprintf(w, `"75%%":%12.2f,`, p[1])
+	fmt.Fprintf(w, `"95%%":%12.2f,`, p[2])
+	fmt.Fprintf(w, `"99%%":%12.2f,`, p[3])
+	fmt.Fprintf(w, `"99.9%%":%12.2f},`, p[4])
+	fmt.Fprintf(w, `"rates":{`)
+	fmt.Fprintf(w, `"1-min":%12.2f,`, t.Rate1())
+	fmt.Fprintf(w, `"5-min":%12.2f,`, t.Rate5())
+	fmt.Fprintf(w, `"15-min":%12.2f,`, t.Rate15())
+	fmt.Fprintf(w, `"mean":%12.2f}}`, t.RateMean())
+}
+
+func WriteTimerCSVHeader(w io.Writer, prefix string) {
+	fmt.Fprintf(w, "%s-count,", prefix)
+	fmt.Fprintf(w, "%s-min,", prefix)
+	fmt.Fprintf(w, "%s-max,", prefix)
+	fmt.Fprintf(w, "%s-mean,", prefix)
+	fmt.Fprintf(w, "%s-stddev,", prefix)
+	fmt.Fprintf(w, "%s-percentile-50%%,", prefix)
+	fmt.Fprintf(w, "%s-percentile-75%%,", prefix)
+	fmt.Fprintf(w, "%s-percentile-95%%,", prefix)
+	fmt.Fprintf(w, "%s-percentile-99%%,", prefix)
+	fmt.Fprintf(w, "%s-percentile-99.9%%,", prefix)
+	fmt.Fprintf(w, "%s-rate-1-min,", prefix)
+	fmt.Fprintf(w, "%s-rate-5-min,", prefix)
+	fmt.Fprintf(w, "%s-rate-15-min,", prefix)
+	fmt.Fprintf(w, "%s-rate-mean", prefix)
+}
+
+func WriteTimerCSV(w io.Writer, timer metrics.Timer) {
+	t := timer.Snapshot()
+	p := t.Percentiles(timerPercentiles)
+
+	fmt.Fprintf(w, `%d,`, t.Count())
+	fmt.Fprintf(w, `%d,`, t.Min())
+	fmt.Fprintf(w, `%d,`, t.Max())
+	fmt.Fprintf(w, `%f,`, t.Mean())
+	fmt.Fprintf(w, `%f,`, t.StdDev())
+	fmt.Fprintf(w, `%f,`, p[0])
+	fmt.Fprintf(w, `%f,`, p[1])
+	fmt.Fprintf(w, `%f,`, p[2])
+	fmt.Fprintf(w, `%f,`, p[3])
+	fmt.Fprintf(w, `%f,`, p[4])
+	fmt.Fprintf(w, `%f,`, t.Rate1())
+	fmt.Fprintf(w, `%f,`, t.Rate5())
+	fmt.Fprintf(w, `%f,`, t.Rate15())
+	fmt.Fprintf(w, `%f`, t.RateMean())
+}

vendor/github.com/blevesearch/bleve/index/store/metrics/metrics_test.go

@@ -0,0 +1,368 @@
+//  Copyright (c) 2015 Couchbase, Inc.
+
+//  Licensed under the Apache License, Version 2.0 (the "License");
+//  you may not use this file except in compliance with the
+//  License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing,
+//  software distributed under the License is distributed on an "AS
+//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+//  express or implied. See the License for the specific language
+//  governing permissions and limitations under the License.
+
+// +build debug
+
+package metrics
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+	_ "github.com/blevesearch/bleve/index/store/gtreap"
+)
+
+func TestMetricsStore(t *testing.T) {
+	s, err := StoreConstructor(map[string]interface{}{})
+	if err == nil {
+		t.Errorf("expected err when bad config")
+	}
+
+	s, err = StoreConstructor(map[string]interface{}{
+		"kvStoreName_actual": "some-invalid-kvstore-name",
+	})
+	if err == nil {
+		t.Errorf("expected err when unknown kvStoreName_actual")
+	}
+
+	s, err = StoreConstructor(map[string]interface{}{
+		"kvStoreName_actual": "gtreap",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	CommonTestKVStore(t, s)
+
+	b := bytes.NewBuffer(nil)
+	s.(*Store).WriteJSON(b)
+	if b.Len() <= 0 {
+		t.Errorf("expected some output from WriteJSON")
+	}
+	var m map[string]interface{}
+	err = json.Unmarshal(b.Bytes(), &m)
+	if err != nil {
+		t.Errorf("expected WriteJSON to be unmarshallable")
+	}
+	if len(m) <= 0 {
+		t.Errorf("expected some entries")
+	}
+
+	b = bytes.NewBuffer(nil)
+	s.(*Store).WriteCSVHeader(b)
+	if b.Len() <= 0 {
+		t.Errorf("expected some output from WriteCSVHeader")
+	}
+
+	b = bytes.NewBuffer(nil)
+	s.(*Store).WriteCSV(b)
+	if b.Len() <= 0 {
+		t.Errorf("expected some output from WriteCSV")
+	}
+}
+
+func TestReaderIsolation(t *testing.T) {
+	s, err := StoreConstructor(map[string]interface{}{
+		"kvStoreName_actual": "gtreap",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	CommonTestReaderIsolation(t, s)
+}
+
+func CommonTestKVStore(t *testing.T, s store.KVStore) {
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Set([]byte("z"), []byte("val-z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Delete([]byte("z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	batch := writer.NewBatch()
+	batch.Set([]byte("b"), []byte("val-b"))
+	batch.Set([]byte("c"), []byte("val-c"))
+	batch.Set([]byte("d"), []byte("val-d"))
+	batch.Set([]byte("e"), []byte("val-e"))
+	batch.Set([]byte("f"), []byte("val-f"))
+	batch.Set([]byte("g"), []byte("val-g"))
+	batch.Set([]byte("h"), []byte("val-h"))
+	batch.Set([]byte("i"), []byte("val-i"))
+	batch.Set([]byte("j"), []byte("val-j"))
+
+	err = batch.Execute()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	it := reader.Iterator([]byte("b"))
+	key, val, valid := it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "b" {
+		t.Fatalf("expected key b, got %s", key)
+	}
+	if string(val) != "val-b" {
+		t.Fatalf("expected value val-b, got %s", val)
+	}
+
+	it.Next()
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "c" {
+		t.Fatalf("expected key c, got %s", key)
+	}
+	if string(val) != "val-c" {
+		t.Fatalf("expected value val-c, got %s", val)
+	}
+
+	it.Seek([]byte("i"))
+	key, val, valid = it.Current()
+	if !valid {
+		t.Fatalf("valid false, expected true")
+	}
+	if string(key) != "i" {
+		t.Fatalf("expected key i, got %s", key)
+	}
+	if string(val) != "val-i" {
+		t.Fatalf("expected value val-i, got %s", val)
+	}
+
+	err = it.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
+	// insert a kv pair
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// create an isolated reader
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	// verify that we see the value already inserted
+	val, err := reader.Get([]byte("a"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-a")) {
+		t.Errorf("expected val-a, got nil")
+	}
+
+	// verify that an iterator sees it
+	count := 0
+	it := reader.Iterator([]byte{0})
+	defer func() {
+		err := it.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it.Valid() {
+		it.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+
+	// add something after the reader was created
+	writer, err = s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("b"), []byte("val-b"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// ensure that a newer reader sees it
+	newReader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := newReader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	val, err = newReader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if !reflect.DeepEqual(val, []byte("val-b")) {
+		t.Errorf("expected val-b, got nil")
+	}
+
+	// ensure that the director iterator sees it
+	count = 0
+	it2 := newReader.Iterator([]byte{0})
+	defer func() {
+		err := it2.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it2.Valid() {
+		it2.Next()
+		count++
+	}
+	if count != 2 {
+		t.Errorf("expected iterator to see 2, saw %d", count)
+	}
+
+	// but that the isolated reader does not
+	val, err = reader.Get([]byte("b"))
+	if err != nil {
+		t.Error(err)
+	}
+	if val != nil {
+		t.Errorf("expected nil, got %v", val)
+	}
+
+	// and ensure that the iterator on the isolated reader also does not
+	count = 0
+	it3 := reader.Iterator([]byte{0})
+	defer func() {
+		err := it3.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for it3.Valid() {
+		it3.Next()
+		count++
+	}
+	if count != 1 {
+		t.Errorf("expected iterator to see 1, saw %d", count)
+	}
+}
+
+func TestErrors(t *testing.T) {
+	s, err := StoreConstructor(map[string]interface{}{
+		"kvStoreName_actual": "gtreap",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	x, ok := s.(*Store)
+	if !ok {
+		t.Errorf("expecting a Store")
+	}
+
+	x.AddError("foo", fmt.Errorf("Foo"), []byte("fooKey"))
+	x.AddError("bar", fmt.Errorf("Bar"), nil)
+	x.AddError("baz", fmt.Errorf("Baz"), []byte("bazKey"))
+
+	b := bytes.NewBuffer(nil)
+	x.WriteJSON(b)
+
+	var m map[string]interface{}
+	err = json.Unmarshal(b.Bytes(), &m)
+	if err != nil {
+		t.Errorf("expected unmarshallable writeJSON, err: %v, b: %s",
+			err, b.Bytes())
+	}
+
+	errorsi, ok := m["Errors"]
+	if !ok || errorsi == nil {
+		t.Errorf("expected errorsi")
+	}
+	errors, ok := errorsi.([]interface{})
+	if !ok || errors == nil {
+		t.Errorf("expected errorsi is array")
+	}
+	if len(errors) != 3 {
+		t.Errorf("expected errors len 3")
+	}
+
+	e := errors[0].(map[string]interface{})
+	if e["Op"].(string) != "foo" ||
+		e["Err"].(string) != "Foo" ||
+		len(e["Time"].(string)) < 10 ||
+		e["Key"].(string) != "fooKey" {
+		t.Errorf("expected foo, %#v", e)
+	}
+	e = errors[1].(map[string]interface{})
+	if e["Op"].(string) != "bar" ||
+		e["Err"].(string) != "Bar" ||
+		len(e["Time"].(string)) < 10 ||
+		e["Key"].(string) != "" {
+		t.Errorf("expected bar, %#v", e)
+	}
+	e = errors[2].(map[string]interface{})
+	if e["Op"].(string) != "baz" ||
+		e["Err"].(string) != "Baz" ||
+		len(e["Time"].(string)) < 10 ||
+		e["Key"].(string) != "bazKey" {
+		t.Errorf("expected baz, %#v", e)
+	}
+}

vendor/github.com/blevesearch/bleve/index/store/null/null.go

@@ -0,0 +1,173 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package null
+
+import (
+	"github.com/blevesearch/bleve/index/store"
+)
+
+type Store struct{}
+
+func New() (*Store, error) {
+	rv := Store{}
+	return &rv, nil
+}
+
+func (i *Store) Open() error {
+	return nil
+}
+
+func (i *Store) SetMergeOperator(mo store.MergeOperator) {
+
+}
+
+func (i *Store) Close() error {
+	return nil
+}
+
+func (i *Store) iterator(key []byte) store.KVIterator {
+	rv := newIterator(i)
+	return rv
+}
+
+func (i *Store) Reader() (store.KVReader, error) {
+	return newReader(i)
+}
+
+func (i *Store) Writer() (store.KVWriter, error) {
+	return newWriter(i)
+}
+
+func (i *Store) newBatch() store.KVBatch {
+	return newBatch(i)
+}
+
+type Reader struct {
+	store *Store
+}
+
+func newReader(store *Store) (*Reader, error) {
+	return &Reader{
+		store: store,
+	}, nil
+}
+
+func (r *Reader) BytesSafeAfterClose() bool {
+	return true
+}
+
+func (r *Reader) Get(key []byte) ([]byte, error) {
+	return nil, nil
+}
+
+func (r *Reader) Iterator(key []byte) store.KVIterator {
+	return r.store.iterator(key)
+}
+
+func (r *Reader) Close() error {
+	return nil
+}
+
+type Iterator struct{}
+
+func newIterator(store *Store) *Iterator {
+	return &Iterator{}
+}
+
+func (i *Iterator) SeekFirst() {}
+
+func (i *Iterator) Seek(k []byte) {}
+
+func (i *Iterator) Next() {}
+
+func (i *Iterator) Current() ([]byte, []byte, bool) {
+	return nil, nil, false
+}
+
+func (i *Iterator) Key() []byte {
+	return nil
+}
+
+func (i *Iterator) Value() []byte {
+	return nil
+}
+
+func (i *Iterator) Valid() bool {
+	return false
+}
+
+func (i *Iterator) Close() error {
+	return nil
+}
+
+type Batch struct{}
+
+func newBatch(s *Store) *Batch {
+	rv := Batch{}
+	return &rv
+}
+
+func (i *Batch) Set(key, val []byte) {
+}
+
+func (i *Batch) Delete(key []byte) {
+}
+
+func (i *Batch) Merge(key, val []byte) {
+}
+
+func (i *Batch) Execute() error {
+	return nil
+}
+
+func (i *Batch) Close() error {
+	return nil
+}
+
+type Writer struct {
+	store *Store
+}
+
+func newWriter(store *Store) (*Writer, error) {
+	return &Writer{
+		store: store,
+	}, nil
+}
+
+func (w *Writer) BytesSafeAfterClose() bool {
+	return true
+}
+
+func (w *Writer) Set(key, val []byte) error {
+	return nil
+}
+
+func (w *Writer) Delete(key []byte) error {
+	return nil
+}
+
+func (w *Writer) NewBatch() store.KVBatch {
+	return newBatch(w.store)
+}
+
+func (w *Writer) Close() error {
+	return nil
+}
+
+// these two methods can safely read using the regular
+// methods without a read transaction, because we know
+// that no one else is writing but us
+func (w *Writer) Get(key []byte) ([]byte, error) {
+	return nil, nil
+}
+
+func (w *Writer) Iterator(key []byte) store.KVIterator {
+	return w.store.iterator(key)
+}

vendor/github.com/blevesearch/bleve/index/store/null/null_test.go

@@ -0,0 +1,88 @@
+package null
+
+import (
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+)
+
+func TestStore(t *testing.T) {
+	s, err := New()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	CommonTestKVStore(t, s)
+}
+
+func CommonTestKVStore(t *testing.T, s store.KVStore) {
+
+	writer, err := s.Writer()
+	if err != nil {
+		t.Error(err)
+	}
+	err = writer.Set([]byte("a"), []byte("val-a"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Set([]byte("z"), []byte("val-z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Delete([]byte("z"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	batch := writer.NewBatch()
+	batch.Set([]byte("b"), []byte("val-b"))
+	batch.Set([]byte("c"), []byte("val-c"))
+	batch.Set([]byte("d"), []byte("val-d"))
+	batch.Set([]byte("e"), []byte("val-e"))
+	batch.Set([]byte("f"), []byte("val-f"))
+	batch.Set([]byte("g"), []byte("val-g"))
+	batch.Set([]byte("h"), []byte("val-h"))
+	batch.Set([]byte("i"), []byte("val-i"))
+	batch.Set([]byte("j"), []byte("val-j"))
+
+	err = batch.Execute()
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err := s.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+	it := reader.Iterator([]byte("b"))
+	key, val, valid := it.Current()
+	if valid {
+		t.Fatalf("valid true, expected false")
+	}
+	if key != nil {
+		t.Fatalf("expected key nil, got %s", key)
+	}
+	if val != nil {
+		t.Fatalf("expected value nil, got %s", val)
+	}
+
+	err = it.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = s.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+}

vendor/github.com/blevesearch/bleve/index/upside_down/analysis_pool.go

@@ -0,0 +1,121 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"github.com/blevesearch/bleve/document"
+)
+
+type AnalysisResult struct {
+	docID string
+	rows  []UpsideDownCouchRow
+}
+
+type AnalysisWork struct {
+	udc *UpsideDownCouch
+	d   *document.Document
+	rc  chan *AnalysisResult
+}
+
+type AnalysisQueue struct {
+	queue chan *AnalysisWork
+	done  chan struct{}
+}
+
+func (q *AnalysisQueue) Queue(work *AnalysisWork) {
+	q.queue <- work
+}
+
+func (q *AnalysisQueue) Close() {
+	close(q.done)
+}
+
+func NewAnalysisQueue(numWorkers int) *AnalysisQueue {
+	rv := AnalysisQueue{
+		queue: make(chan *AnalysisWork),
+		done:  make(chan struct{}),
+	}
+	for i := 0; i < numWorkers; i++ {
+		go AnalysisWorker(rv)
+	}
+	return &rv
+}
+
+func AnalysisWorker(q AnalysisQueue) {
+	// read work off the queue
+	for {
+		select {
+		case <-q.done:
+			return
+		case w := <-q.queue:
+
+			rv := &AnalysisResult{
+				docID: w.d.ID,
+				rows:  make([]UpsideDownCouchRow, 0, 100),
+			}
+
+			// track our back index entries
+			backIndexTermEntries := make([]*BackIndexTermEntry, 0)
+			backIndexStoredEntries := make([]*BackIndexStoreEntry, 0)
+
+			for _, field := range w.d.Fields {
+				fieldIndex, newFieldRow := w.udc.fieldIndexCache.FieldIndex(field.Name())
+				if newFieldRow != nil {
+					rv.rows = append(rv.rows, newFieldRow)
+				}
+
+				if field.Options().IsIndexed() {
+
+					fieldLength, tokenFreqs := field.Analyze()
+
+					// see if any of the composite fields need this
+					for _, compositeField := range w.d.CompositeFields {
+						compositeField.Compose(field.Name(), fieldLength, tokenFreqs)
+					}
+
+					// encode this field
+					indexRows, indexBackIndexTermEntries := w.udc.indexField(w.d.ID, field, fieldIndex, fieldLength, tokenFreqs)
+					rv.rows = append(rv.rows, indexRows...)
+					backIndexTermEntries = append(backIndexTermEntries, indexBackIndexTermEntries...)
+				}
+
+				if field.Options().IsStored() {
+					storeRows, indexBackIndexStoreEntries := w.udc.storeField(w.d.ID, field, fieldIndex)
+					rv.rows = append(rv.rows, storeRows...)
+					backIndexStoredEntries = append(backIndexStoredEntries, indexBackIndexStoreEntries...)
+				}
+
+			}
+
+			// now index the composite fields
+			for _, compositeField := range w.d.CompositeFields {
+				fieldIndex, newFieldRow := w.udc.fieldIndexCache.FieldIndex(compositeField.Name())
+				if newFieldRow != nil {
+					rv.rows = append(rv.rows, newFieldRow)
+				}
+				if compositeField.Options().IsIndexed() {
+					fieldLength, tokenFreqs := compositeField.Analyze()
+					// encode this field
+					indexRows, indexBackIndexTermEntries := w.udc.indexField(w.d.ID, compositeField, fieldIndex, fieldLength, tokenFreqs)
+					rv.rows = append(rv.rows, indexRows...)
+					backIndexTermEntries = append(backIndexTermEntries, indexBackIndexTermEntries...)
+				}
+			}
+
+			// build the back index row
+			backIndexRow := NewBackIndexRow(w.d.ID, backIndexTermEntries, backIndexStoredEntries)
+			rv.rows = append(rv.rows, backIndexRow)
+
+			w.rc <- rv
+
+		}
+
+	}
+}

vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_all.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+BENCHMARKS=`grep "func Benchmark" *_test.go  | sed 's/.*func //' | sed s/\(.*{//`
+
+for BENCHMARK in $BENCHMARKS
+do
+	go test -v -run=xxx -bench=^$BENCHMARK$ -benchtime=10s -tags 'forestdb leveldb'  | grep -v ok | grep -v PASS
+done

vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_boltdb_test.go

@@ -0,0 +1,77 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"os"
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/index/store/boltdb"
+)
+
+func CreateBoltDB() (store.KVStore, error) {
+	s := boltdb.New("test", "bleve")
+	return s, nil
+}
+
+func DestroyBoltDB() error {
+	return os.RemoveAll("test")
+}
+
+func BenchmarkBoltDBIndexing1Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateBoltDB, DestroyBoltDB, 1)
+}
+
+func BenchmarkBoltDBIndexing2Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateBoltDB, DestroyBoltDB, 2)
+}
+
+func BenchmarkBoltDBIndexing4Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateBoltDB, DestroyBoltDB, 4)
+}
+
+// batches
+
+func BenchmarkBoltDBIndexing1Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 1, 10)
+}
+
+func BenchmarkBoltDBIndexing2Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 2, 10)
+}
+
+func BenchmarkBoltDBIndexing4Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 4, 10)
+}
+
+func BenchmarkBoltDBIndexing1Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 1, 100)
+}
+
+func BenchmarkBoltDBIndexing2Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 2, 100)
+}
+
+func BenchmarkBoltDBIndexing4Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 4, 100)
+}
+
+func BenchmarkBoltBIndexing1Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 1, 1000)
+}
+
+func BenchmarkBoltBIndexing2Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 2, 1000)
+}
+
+func BenchmarkBoltBIndexing4Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateBoltDB, DestroyBoltDB, 4, 1000)
+}

vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_common_test.go

@@ -0,0 +1,143 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"strconv"
+	"testing"
+
+	_ "github.com/blevesearch/bleve/analysis/analyzers/standard_analyzer"
+	"github.com/blevesearch/bleve/document"
+	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/registry"
+)
+
+var benchmarkDocBodies = []string{
+	"A boiling liquid expanding vapor explosion (BLEVE, /ˈblɛviː/ blev-ee) is an explosion caused by the rupture of a vessel containing a pressurized liquid above its boiling point.",
+	"A boiler explosion is a catastrophic failure of a boiler. As seen today, boiler explosions are of two kinds. One kind is a failure of the pressure parts of the steam and water sides. There can be many different causes, such as failure of the safety valve, corrosion of critical parts of the boiler, or low water level. Corrosion along the edges of lap joints was a common cause of early boiler explosions.",
+	"A boiler is a closed vessel in which water or other fluid is heated. The fluid does not necessarily boil. (In North America the term \"furnace\" is normally used if the purpose is not actually to boil the fluid.) The heated or vaporized fluid exits the boiler for use in various processes or heating applications,[1][2] including central heating, boiler-based power generation, cooking, and sanitation.",
+	"A pressure vessel is a closed container designed to hold gases or liquids at a pressure substantially different from the ambient pressure.",
+	"Pressure (symbol: p or P) is the ratio of force to the area over which that force is distributed.",
+	"Liquid is one of the four fundamental states of matter (the others being solid, gas, and plasma), and is the only state with a definite volume but no fixed shape.",
+	"The boiling point of a substance is the temperature at which the vapor pressure of the liquid equals the pressure surrounding the liquid[1][2] and the liquid changes into a vapor.",
+	"Vapor pressure or equilibrium vapor pressure is defined as the pressure exerted by a vapor in thermodynamic equilibrium with its condensed phases (solid or liquid) at a given temperature in a closed system.",
+	"Industrial gases are a group of gases that are specifically manufactured for use in a wide range of industries, which include oil and gas, petrochemicals, chemicals, power, mining, steelmaking, metals, environmental protection, medicine, pharmaceuticals, biotechnology, food, water, fertilizers, nuclear power, electronics and aerospace.",
+	"The expansion ratio of a liquefied and cryogenic substance is the volume of a given amount of that substance in liquid form compared to the volume of the same amount of substance in gaseous form, at room temperature and normal atmospheric pressure.",
+}
+
+type KVStoreCreate func() (store.KVStore, error)
+type KVStoreDestroy func() error
+
+func CommonBenchmarkIndex(b *testing.B, create KVStoreCreate, destroy KVStoreDestroy, analysisWorkers int) {
+
+	cache := registry.NewCache()
+	analyzer, err := cache.AnalyzerNamed("standard")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	indexDocument := document.NewDocument("").
+		AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[0]), analyzer))
+
+	b.ResetTimer()
+	b.StopTimer()
+	for i := 0; i < b.N; i++ {
+		s, err := create()
+		if err != nil {
+			b.Fatal(err)
+		}
+		analysisQueue := NewAnalysisQueue(analysisWorkers)
+		idx := NewUpsideDownCouch(s, analysisQueue)
+
+		err = idx.Open()
+		if err != nil {
+			b.Fatal(err)
+		}
+		indexDocument.ID = strconv.Itoa(i)
+		// just time the indexing portion
+		b.StartTimer()
+		err = idx.Update(indexDocument)
+		if err != nil {
+			b.Fatal(err)
+		}
+		b.StopTimer()
+		err = idx.Close()
+		if err != nil {
+			b.Fatal(err)
+		}
+		err = destroy()
+		if err != nil {
+			b.Fatal(err)
+		}
+		analysisQueue.Close()
+	}
+}
+
+func CommonBenchmarkIndexBatch(b *testing.B, create KVStoreCreate, destroy KVStoreDestroy, analysisWorkers, batchSize int) {
+
+	cache := registry.NewCache()
+	analyzer, err := cache.AnalyzerNamed("standard")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+	b.StopTimer()
+	for i := 0; i < b.N; i++ {
+
+		s, err := create()
+		if err != nil {
+			b.Fatal(err)
+		}
+		analysisQueue := NewAnalysisQueue(analysisWorkers)
+		idx := NewUpsideDownCouch(s, analysisQueue)
+
+		err = idx.Open()
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		b.StartTimer()
+		batch := index.NewBatch()
+		for j := 0; j < 1000; j++ {
+			if j%batchSize == 0 {
+				if len(batch.IndexOps) > 0 {
+					err := idx.Batch(batch)
+					if err != nil {
+						b.Fatal(err)
+					}
+				}
+				batch = index.NewBatch()
+			}
+			indexDocument := document.NewDocument("").
+				AddField(document.NewTextFieldWithAnalyzer("body", []uint64{}, []byte(benchmarkDocBodies[j%10]), analyzer))
+			indexDocument.ID = strconv.Itoa(i) + "-" + strconv.Itoa(j)
+			batch.Update(indexDocument)
+		}
+		// close last batch
+		if len(batch.IndexOps) > 0 {
+			err := idx.Batch(batch)
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+		b.StopTimer()
+		err = idx.Close()
+		if err != nil {
+			b.Fatal(err)
+		}
+		err = destroy()
+		if err != nil {
+			b.Fatal(err)
+		}
+		analysisQueue.Close()
+	}
+}

vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_cznicb_test.go

@@ -0,0 +1,75 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/index/store/cznicb"
+)
+
+func CreateCznicB() (store.KVStore, error) {
+	return cznicb.StoreConstructor(nil)
+}
+
+func DestroyCznicB() error {
+	return nil
+}
+
+func BenchmarkCznicBIndexing1Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateCznicB, DestroyCznicB, 1)
+}
+
+func BenchmarkCznicBIndexing2Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateCznicB, DestroyCznicB, 2)
+}
+
+func BenchmarkCznicBIndexing4Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateCznicB, DestroyCznicB, 4)
+}
+
+// batches
+
+func BenchmarkCznicBIndexing1Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 1, 10)
+}
+
+func BenchmarkCznicBIndexing2Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 2, 10)
+}
+
+func BenchmarkCznicBIndexing4Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 4, 10)
+}
+
+func BenchmarkCznicBIndexing1Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 1, 100)
+}
+
+func BenchmarkCznicBIndexing2Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 2, 100)
+}
+
+func BenchmarkCznicBIndexing4Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 4, 100)
+}
+
+func BenchmarkCznicBIndexing1Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 1, 1000)
+}
+
+func BenchmarkCznicBIndexing2Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 2, 1000)
+}
+
+func BenchmarkCznicBIndexing4Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateCznicB, DestroyCznicB, 4, 1000)
+}

vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_forestdb_test.go

@@ -0,0 +1,86 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build forestdb
+
+package upside_down
+
+import (
+	"os"
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/index/store/forestdb"
+)
+
+func CreateForestDB() (store.KVStore, error) {
+	err := os.MkdirAll("testdir", 0700)
+	if err != nil {
+		return nil, err
+	}
+	s, err := forestdb.New("testdir/test", true, nil)
+	if err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+func DestroyForestDB() error {
+	return os.RemoveAll("testdir")
+}
+
+func BenchmarkForestDBIndexing1Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateForestDB, DestroyForestDB, 1)
+}
+
+func BenchmarkForestDBIndexing2Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateForestDB, DestroyForestDB, 2)
+}
+
+func BenchmarkForestDBIndexing4Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateForestDB, DestroyForestDB, 4)
+}
+
+// batches
+
+func BenchmarkForestDBIndexing1Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 1, 10)
+}
+
+func BenchmarkForestDBIndexing2Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 2, 10)
+}
+
+func BenchmarkForestDBIndexing4Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 4, 10)
+}
+
+func BenchmarkForestDBIndexing1Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 1, 100)
+}
+
+func BenchmarkForestDBIndexing2Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 2, 100)
+}
+
+func BenchmarkForestDBIndexing4Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 4, 100)
+}
+
+func BenchmarkForestDBIndexing1Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 1, 1000)
+}
+
+func BenchmarkForestDBIndexing2Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 2, 1000)
+}
+
+func BenchmarkForestDBIndexing4Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateForestDB, DestroyForestDB, 4, 1000)
+}

vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_goleveldb_test.go

@@ -0,0 +1,80 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"os"
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/index/store/goleveldb"
+)
+
+var goLevelDBTestOptions = map[string]interface{}{
+	"create_if_missing": true,
+}
+
+func CreateGoLevelDB() (store.KVStore, error) {
+	return goleveldb.New("test", goLevelDBTestOptions)
+}
+
+func DestroyGoLevelDB() error {
+	return os.RemoveAll("test")
+}
+
+func BenchmarkGoLevelDBIndexing1Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateGoLevelDB, DestroyGoLevelDB, 1)
+}
+
+func BenchmarkGoLevelDBIndexing2Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateGoLevelDB, DestroyGoLevelDB, 2)
+}
+
+func BenchmarkGoLevelDBIndexing4Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateGoLevelDB, DestroyGoLevelDB, 4)
+}
+
+// batches
+
+func BenchmarkGoLevelDBIndexing1Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 1, 10)
+}
+
+func BenchmarkGoLevelDBIndexing2Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 2, 10)
+}
+
+func BenchmarkGoLevelDBIndexing4Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 4, 10)
+}
+
+func BenchmarkGoLevelDBIndexing1Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 1, 100)
+}
+
+func BenchmarkGoLevelDBIndexing2Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 2, 100)
+}
+
+func BenchmarkGoLevelDBIndexing4Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 4, 100)
+}
+
+func BenchmarkGoLevelDBIndexing1Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 1, 1000)
+}
+
+func BenchmarkGoLevelDBIndexing2Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 2, 1000)
+}
+
+func BenchmarkGoLevelDBIndexing4Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoLevelDB, DestroyGoLevelDB, 4, 1000)
+}

vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_gorocksdb_test.go

@@ -0,0 +1,82 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build rocksdb
+
+package upside_down
+
+import (
+	"os"
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/index/store/gorocksdb"
+)
+
+var rocksdbTestOptions = map[string]interface{}{
+	"create_if_missing": true,
+}
+
+func CreateGoRocksDB() (store.KVStore, error) {
+	return rocksdb.New("test", rocksdbTestOptions)
+}
+
+func DestroyGoRocksDB() error {
+	return os.RemoveAll("test")
+}
+
+func BenchmarkRocksDBIndexing1Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateGoRocksDB, DestroyGoRocksDB, 1)
+}
+
+func BenchmarkRocksDBIndexing2Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateGoRocksDB, DestroyGoRocksDB, 2)
+}
+
+func BenchmarkRocksDBIndexing4Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateGoRocksDB, DestroyGoRocksDB, 4)
+}
+
+// batches
+
+func BenchmarkRocksDBIndexing1Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 1, 10)
+}
+
+func BenchmarkRocksDBIndexing2Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 2, 10)
+}
+
+func BenchmarkRocksDBIndexing4Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 4, 10)
+}
+
+func BenchmarkRocksDBIndexing1Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 1, 100)
+}
+
+func BenchmarkRocksDBIndexing2Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 2, 100)
+}
+
+func BenchmarkRocksDBIndexing4Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 4, 100)
+}
+
+func BenchmarkRocksDBIndexing1Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 1, 1000)
+}
+
+func BenchmarkRocksDBIndexing2Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 2, 1000)
+}
+
+func BenchmarkRocksDBIndexing4Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGoRocksDB, DestroyGoRocksDB, 4, 1000)
+}

vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_gtreap_test.go

@@ -0,0 +1,75 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/index/store/gtreap"
+)
+
+func CreateGTreap() (store.KVStore, error) {
+	return gtreap.StoreConstructor(nil)
+}
+
+func DestroyGTreap() error {
+	return nil
+}
+
+func BenchmarkGTreapIndexing1Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateGTreap, DestroyGTreap, 1)
+}
+
+func BenchmarkGTreapIndexing2Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateGTreap, DestroyGTreap, 2)
+}
+
+func BenchmarkGTreapIndexing4Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateGTreap, DestroyGTreap, 4)
+}
+
+// batches
+
+func BenchmarkGTreapIndexing1Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 1, 10)
+}
+
+func BenchmarkGTreapIndexing2Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 2, 10)
+}
+
+func BenchmarkGTreapIndexing4Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 4, 10)
+}
+
+func BenchmarkGTreapIndexing1Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 1, 100)
+}
+
+func BenchmarkGTreapIndexing2Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 2, 100)
+}
+
+func BenchmarkGTreapIndexing4Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 4, 100)
+}
+
+func BenchmarkGTreapIndexing1Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 1, 1000)
+}
+
+func BenchmarkGTreapIndexing2Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 2, 1000)
+}
+
+func BenchmarkGTreapIndexing4Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateGTreap, DestroyGTreap, 4, 1000)
+}

vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_inmem_test.go

@@ -0,0 +1,75 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/index/store/inmem"
+)
+
+func CreateInMem() (store.KVStore, error) {
+	return inmem.New()
+}
+
+func DestroyInMem() error {
+	return nil
+}
+
+func BenchmarkInMemIndexing1Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateInMem, DestroyInMem, 1)
+}
+
+func BenchmarkInMemIndexing2Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateInMem, DestroyInMem, 2)
+}
+
+func BenchmarkInMemIndexing4Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateInMem, DestroyInMem, 4)
+}
+
+// batches
+
+func BenchmarkInMemIndexing1Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 1, 10)
+}
+
+func BenchmarkInMemIndexing2Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 2, 10)
+}
+
+func BenchmarkInMemIndexing4Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 4, 10)
+}
+
+func BenchmarkInMemIndexing1Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 1, 100)
+}
+
+func BenchmarkInMemIndexing2Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 2, 100)
+}
+
+func BenchmarkInMemIndexing4Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 4, 100)
+}
+
+func BenchmarkInMemIndexing1Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 1, 1000)
+}
+
+func BenchmarkInMemIndexing2Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 2, 1000)
+}
+
+func BenchmarkInMemIndexing4Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateInMem, DestroyInMem, 4, 1000)
+}

vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_leveldb_test.go

@@ -0,0 +1,82 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+// +build leveldb full
+
+package upside_down
+
+import (
+	"os"
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/index/store/leveldb"
+)
+
+var leveldbTestOptions = map[string]interface{}{
+	"create_if_missing": true,
+}
+
+func CreateLevelDB() (store.KVStore, error) {
+	return leveldb.New("test", leveldbTestOptions)
+}
+
+func DestroyLevelDB() error {
+	return os.RemoveAll("test")
+}
+
+func BenchmarkLevelDBIndexing1Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateLevelDB, DestroyLevelDB, 1)
+}
+
+func BenchmarkLevelDBIndexing2Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateLevelDB, DestroyLevelDB, 2)
+}
+
+func BenchmarkLevelDBIndexing4Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateLevelDB, DestroyLevelDB, 4)
+}
+
+// batches
+
+func BenchmarkLevelDBIndexing1Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 1, 10)
+}
+
+func BenchmarkLevelDBIndexing2Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 2, 10)
+}
+
+func BenchmarkLevelDBIndexing4Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 4, 10)
+}
+
+func BenchmarkLevelDBIndexing1Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 1, 100)
+}
+
+func BenchmarkLevelDBIndexing2Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 2, 100)
+}
+
+func BenchmarkLevelDBIndexing4Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 4, 100)
+}
+
+func BenchmarkLevelDBIndexing1Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 1, 1000)
+}
+
+func BenchmarkLevelDBIndexing2Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 2, 1000)
+}
+
+func BenchmarkLevelDBIndexing4Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateLevelDB, DestroyLevelDB, 4, 1000)
+}

vendor/github.com/blevesearch/bleve/index/upside_down/benchmark_null_test.go

@@ -0,0 +1,75 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"testing"
+
+	"github.com/blevesearch/bleve/index/store"
+	"github.com/blevesearch/bleve/index/store/null"
+)
+
+func CreateNull() (store.KVStore, error) {
+	return null.New()
+}
+
+func DestroyNull() error {
+	return nil
+}
+
+func BenchmarkNullIndexing1Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateNull, DestroyNull, 1)
+}
+
+func BenchmarkNullIndexing2Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateNull, DestroyNull, 2)
+}
+
+func BenchmarkNullIndexing4Workers(b *testing.B) {
+	CommonBenchmarkIndex(b, CreateNull, DestroyNull, 4)
+}
+
+// batches
+
+func BenchmarkNullIndexing1Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 1, 10)
+}
+
+func BenchmarkNullIndexing2Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 2, 10)
+}
+
+func BenchmarkNullIndexing4Workers10Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 4, 10)
+}
+
+func BenchmarkNullIndexing1Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 1, 100)
+}
+
+func BenchmarkNullIndexing2Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 2, 100)
+}
+
+func BenchmarkNullIndexing4Workers100Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 4, 100)
+}
+
+func BenchmarkNullIndexing1Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 1, 1000)
+}
+
+func BenchmarkNullIndexing2Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 2, 1000)
+}
+
+func BenchmarkNullIndexing4Workers1000Batch(b *testing.B) {
+	CommonBenchmarkIndexBatch(b, CreateNull, DestroyNull, 4, 1000)
+}

vendor/github.com/blevesearch/bleve/index/upside_down/dump.go

@@ -0,0 +1,177 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"bytes"
+	"sort"
+
+	"github.com/blevesearch/bleve/index/store"
+)
+
+// the functions in this file are only intended to be used by
+// the bleve_dump utility and the debug http handlers
+// if your application relies on them, you're doing something wrong
+// they may change or be removed at any time
+
+func (udc *UpsideDownCouch) dumpPrefix(kvreader store.KVReader, rv chan interface{}, prefix []byte) {
+	start := prefix
+	if start == nil {
+		start = []byte{0}
+	}
+	it := kvreader.Iterator(start)
+	defer func() {
+		cerr := it.Close()
+		if cerr != nil {
+			rv <- cerr
+		}
+	}()
+	key, val, valid := it.Current()
+	for valid {
+
+		if prefix != nil && !bytes.HasPrefix(key, prefix) {
+			break
+		}
+
+		row, err := ParseFromKeyValue(key, val)
+		if err != nil {
+			rv <- err
+			return
+		}
+		rv <- row
+
+		it.Next()
+		key, val, valid = it.Current()
+	}
+}
+
+func (udc *UpsideDownCouch) DumpAll() chan interface{} {
+	rv := make(chan interface{})
+	go func() {
+		defer close(rv)
+
+		// start an isolated reader for use during the dump
+		kvreader, err := udc.store.Reader()
+		if err != nil {
+			rv <- err
+			return
+		}
+		defer func() {
+			cerr := kvreader.Close()
+			if cerr != nil {
+				rv <- cerr
+			}
+		}()
+
+		udc.dumpPrefix(kvreader, rv, nil)
+	}()
+	return rv
+}
+
+func (udc *UpsideDownCouch) DumpFields() chan interface{} {
+	rv := make(chan interface{})
+	go func() {
+		defer close(rv)
+
+		// start an isolated reader for use during the dump
+		kvreader, err := udc.store.Reader()
+		if err != nil {
+			rv <- err
+			return
+		}
+		defer func() {
+			cerr := kvreader.Close()
+			if cerr != nil {
+				rv <- cerr
+			}
+		}()
+
+		udc.dumpPrefix(kvreader, rv, []byte{'f'})
+	}()
+	return rv
+}
+
+type keyset [][]byte
+
+func (k keyset) Len() int           { return len(k) }
+func (k keyset) Swap(i, j int)      { k[i], k[j] = k[j], k[i] }
+func (k keyset) Less(i, j int) bool { return bytes.Compare(k[i], k[j]) < 0 }
+
+// DumpDoc returns all rows in the index related to this doc id
+func (udc *UpsideDownCouch) DumpDoc(id string) chan interface{} {
+	rv := make(chan interface{})
+
+	go func() {
+		defer close(rv)
+
+		// start an isolated reader for use during the dump
+		kvreader, err := udc.store.Reader()
+		if err != nil {
+			rv <- err
+			return
+		}
+		defer func() {
+			cerr := kvreader.Close()
+			if cerr != nil {
+				rv <- cerr
+			}
+		}()
+
+		back, err := udc.backIndexRowForDoc(kvreader, id)
+		if err != nil {
+			rv <- err
+			return
+		}
+
+		// no such doc
+		if back == nil {
+			return
+		}
+		// build sorted list of term keys
+		keys := make(keyset, 0)
+		for _, entry := range back.termEntries {
+			tfr := NewTermFrequencyRow([]byte(*entry.Term), uint16(*entry.Field), id, 0, 0)
+			key := tfr.Key()
+			keys = append(keys, key)
+		}
+		sort.Sort(keys)
+
+		// first add all the stored rows
+		storedRowPrefix := NewStoredRow(id, 0, []uint64{}, 'x', []byte{}).ScanPrefixForDoc()
+		udc.dumpPrefix(kvreader, rv, storedRowPrefix)
+
+		// now walk term keys in order and add them as well
+		if len(keys) > 0 {
+			it := kvreader.Iterator(keys[0])
+			defer func() {
+				cerr := it.Close()
+				if cerr != nil {
+					rv <- cerr
+				}
+			}()
+
+			for _, key := range keys {
+				it.Seek(key)
+				rkey, rval, valid := it.Current()
+				if !valid {
+					break
+				}
+				row, err := ParseFromKeyValue(rkey, rval)
+				if err != nil {
+					rv <- err
+					return
+				}
+				rv <- row
+			}
+		}
+	}()
+
+	return rv
+}

vendor/github.com/blevesearch/bleve/index/upside_down/dump_test.go

@@ -0,0 +1,127 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"github.com/blevesearch/bleve/index/store/boltdb"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/blevesearch/bleve/document"
+)
+
+func TestDump(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	s := boltdb.New("test", "bleve")
+	s.SetMergeOperator(&mergeOperator)
+	analysisQueue := NewAnalysisQueue(1)
+	idx := NewUpsideDownCouch(s, analysisQueue)
+	err := idx.Open()
+	if err != nil {
+		t.Errorf("error opening index: %v", err)
+	}
+	defer func() {
+		err := idx.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	var expectedCount uint64
+	docCount, err := idx.DocCount()
+	if err != nil {
+		t.Error(err)
+	}
+	if docCount != expectedCount {
+		t.Errorf("Expected document count to be %d got %d", expectedCount, docCount)
+	}
+
+	doc := document.NewDocument("1")
+	doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test"), document.IndexField|document.StoreField))
+	doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField))
+	dateField, err := document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField)
+	if err != nil {
+		t.Error(err)
+	}
+	doc.AddField(dateField)
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+
+	doc = document.NewDocument("2")
+	doc.AddField(document.NewTextFieldWithIndexingOptions("name", []uint64{}, []byte("test2"), document.IndexField|document.StoreField))
+	doc.AddField(document.NewNumericFieldWithIndexingOptions("age", []uint64{}, 35.99, document.IndexField|document.StoreField))
+	dateField, err = document.NewDateTimeFieldWithIndexingOptions("unixEpoch", []uint64{}, time.Unix(0, 0), document.IndexField|document.StoreField)
+	if err != nil {
+		t.Error(err)
+	}
+	doc.AddField(dateField)
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+
+	fieldsCount := 0
+	fieldsRows := idx.DumpFields()
+	for _ = range fieldsRows {
+		fieldsCount++
+	}
+	if fieldsCount != 3 {
+		t.Errorf("expected 3 fields, got %d", fieldsCount)
+	}
+
+	// 1 text term
+	// 16 numeric terms
+	// 16 date terms
+	// 3 stored fields
+	expectedDocRowCount := int(1 + (2 * (64 / document.DefaultPrecisionStep)) + 3)
+	docRowCount := 0
+	docRows := idx.DumpDoc("1")
+	for _ = range docRows {
+		docRowCount++
+	}
+	if docRowCount != expectedDocRowCount {
+		t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount)
+	}
+
+	docRowCount = 0
+	docRows = idx.DumpDoc("2")
+	for _ = range docRows {
+		docRowCount++
+	}
+	if docRowCount != expectedDocRowCount {
+		t.Errorf("expected %d rows for document, got %d", expectedDocRowCount, docRowCount)
+	}
+
+	// 1 version
+	// fieldsCount field rows
+	// 2 docs * expectedDocRowCount
+	// 2 back index rows
+	// 2 text term row count (2 different text terms)
+	// 16 numeric term row counts (shared for both docs, same numeric value)
+	// 16 date term row counts (shared for both docs, same date value)
+	expectedAllRowCount := int(1 + fieldsCount + (2 * expectedDocRowCount) + 2 + 2 + int((2 * (64 / document.DefaultPrecisionStep))))
+	allRowCount := 0
+	allRows := idx.DumpAll()
+	for _ = range allRows {
+		allRowCount++
+	}
+	if allRowCount != expectedAllRowCount {
+		t.Errorf("expected %d rows for all, got %d", expectedAllRowCount, allRowCount)
+	}
+}

vendor/github.com/blevesearch/bleve/index/upside_down/field_dict.go

@@ -0,0 +1,86 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/index/store"
+)
+
+type UpsideDownCouchFieldDict struct {
+	indexReader *IndexReader
+	iterator    store.KVIterator
+	endKey      []byte
+	field       uint16
+}
+
+func newUpsideDownCouchFieldDict(indexReader *IndexReader, field uint16, startTerm, endTerm []byte) (*UpsideDownCouchFieldDict, error) {
+
+	startKey := NewDictionaryRow(startTerm, field, 0).Key()
+	if endTerm == nil {
+		endTerm = []byte{ByteSeparator}
+	}
+	endKey := NewDictionaryRow(endTerm, field, 0).Key()
+
+	it := indexReader.kvreader.Iterator(startKey)
+
+	return &UpsideDownCouchFieldDict{
+		indexReader: indexReader,
+		iterator:    it,
+		field:       field,
+		endKey:      endKey,
+	}, nil
+
+}
+
+func (r *UpsideDownCouchFieldDict) Next() (*index.DictEntry, error) {
+	key, val, valid := r.iterator.Current()
+	if !valid {
+		return nil, nil
+	}
+
+	// past end term
+	if bytes.Compare(key, r.endKey) > 0 {
+		return nil, nil
+	}
+
+	currRow, err := NewDictionaryRowKV(key, val)
+	if err != nil {
+		return nil, fmt.Errorf("unexpected error parsing dictionary row kv: %v", err)
+	}
+	rv := index.DictEntry{
+		Term:  string(currRow.term),
+		Count: currRow.count,
+	}
+	// advance the iterator to the next term
+	r.iterator.Next()
+	return &rv, nil
+
+}
+
+func (r *UpsideDownCouchFieldDict) Close() error {
+	return r.iterator.Close()
+}
+
+func incrementBytes(in []byte) []byte {
+	rv := make([]byte, len(in))
+	copy(rv, in)
+	for i := len(rv) - 1; i >= 0; i-- {
+		rv[i] = rv[i] + 1
+		if rv[i] != 0 {
+			// didn't overflow, so stop
+			break
+		}
+	}
+	return rv
+}

vendor/github.com/blevesearch/bleve/index/upside_down/field_dict_test.go

@@ -0,0 +1,180 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/document"
+	"github.com/blevesearch/bleve/index/store/boltdb"
+)
+
+func TestIndexFieldDict(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	s := boltdb.New("test", "bleve")
+	s.SetMergeOperator(&mergeOperator)
+	analysisQueue := NewAnalysisQueue(1)
+	idx := NewUpsideDownCouch(s, analysisQueue)
+	err := idx.Open()
+	if err != nil {
+		t.Errorf("error opening index: %v", err)
+	}
+	defer func() {
+		err := idx.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	var expectedCount uint64
+	doc := document.NewDocument("1")
+	doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+	expectedCount++
+
+	doc = document.NewDocument("2")
+	doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test test test"), testAnalyzer))
+	doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), document.IndexField|document.IncludeTermVectors, testAnalyzer))
+	doc.AddField(document.NewTextFieldCustom("prefix", []uint64{}, []byte("bob cat cats catting dog doggy zoo"), document.IndexField|document.IncludeTermVectors, testAnalyzer))
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+	expectedCount++
+
+	indexReader, err := idx.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := indexReader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	dict, err := indexReader.FieldDict("name")
+	if err != nil {
+		t.Errorf("error creating reader: %v", err)
+	}
+	defer func() {
+		err := dict.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	termCount := 0
+	curr, err := dict.Next()
+	for err == nil && curr != nil {
+		termCount++
+		if curr.Term != "test" {
+			t.Errorf("expected term to be 'test', got '%s'", curr.Term)
+		}
+		curr, err = dict.Next()
+	}
+	if termCount != 1 {
+		t.Errorf("expected 1 term for this field, got %d", termCount)
+	}
+
+	dict2, err := indexReader.FieldDict("desc")
+	if err != nil {
+		t.Errorf("error creating reader: %v", err)
+	}
+	defer func() {
+		err := dict2.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	termCount = 0
+	terms := make([]string, 0)
+	curr, err = dict2.Next()
+	for err == nil && curr != nil {
+		termCount++
+		terms = append(terms, curr.Term)
+		curr, err = dict2.Next()
+	}
+	if termCount != 3 {
+		t.Errorf("expected 3 term for this field, got %d", termCount)
+	}
+	expectedTerms := []string{"eat", "more", "rice"}
+	if !reflect.DeepEqual(expectedTerms, terms) {
+		t.Errorf("expected %#v, got %#v", expectedTerms, terms)
+	}
+
+	// test start and end range
+	dict3, err := indexReader.FieldDictRange("desc", []byte("fun"), []byte("nice"))
+	if err != nil {
+		t.Errorf("error creating reader: %v", err)
+	}
+	defer func() {
+		err := dict3.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	termCount = 0
+	terms = make([]string, 0)
+	curr, err = dict3.Next()
+	for err == nil && curr != nil {
+		termCount++
+		terms = append(terms, curr.Term)
+		curr, err = dict3.Next()
+	}
+	if termCount != 1 {
+		t.Errorf("expected 1 term for this field, got %d", termCount)
+	}
+	expectedTerms = []string{"more"}
+	if !reflect.DeepEqual(expectedTerms, terms) {
+		t.Errorf("expected %#v, got %#v", expectedTerms, terms)
+	}
+
+	// test use case for prefix
+	dict4, err := indexReader.FieldDictPrefix("prefix", []byte("cat"))
+	if err != nil {
+		t.Errorf("error creating reader: %v", err)
+	}
+	defer func() {
+		err := dict4.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	termCount = 0
+	terms = make([]string, 0)
+	curr, err = dict4.Next()
+	for err == nil && curr != nil {
+		termCount++
+		terms = append(terms, curr.Term)
+		curr, err = dict4.Next()
+	}
+	if termCount != 3 {
+		t.Errorf("expected 3 term for this field, got %d", termCount)
+	}
+	expectedTerms = []string{"cat", "cats", "catting"}
+	if !reflect.DeepEqual(expectedTerms, terms) {
+		t.Errorf("expected %#v, got %#v", expectedTerms, terms)
+	}
+}

vendor/github.com/blevesearch/bleve/index/upside_down/field_index_cache.go

@@ -0,0 +1,69 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"sync"
+)
+
+type FieldIndexCache struct {
+	fieldIndexes   map[string]uint16
+	lastFieldIndex int
+	mutex          sync.RWMutex
+}
+
+func NewFieldIndexCache() *FieldIndexCache {
+	return &FieldIndexCache{
+		fieldIndexes: make(map[string]uint16),
+	}
+}
+
+func (f *FieldIndexCache) AddExisting(field string, index uint16) {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+	f.fieldIndexes[field] = index
+	if int(index) > f.lastFieldIndex {
+		f.lastFieldIndex = int(index)
+	}
+}
+
+func (f *FieldIndexCache) FieldExists(field string) (uint16, bool) {
+	f.mutex.RLock()
+	defer f.mutex.RUnlock()
+	if index, ok := f.fieldIndexes[field]; ok {
+		return index, true
+	}
+	return 0, false
+}
+
+func (f *FieldIndexCache) FieldIndex(field string) (uint16, *FieldRow) {
+	f.mutex.Lock()
+	defer f.mutex.Unlock()
+	index, exists := f.fieldIndexes[field]
+	if exists {
+		return index, nil
+	}
+	// assign next field id
+	index = uint16(f.lastFieldIndex + 1)
+	f.fieldIndexes[field] = index
+	f.lastFieldIndex = int(index)
+	return index, NewFieldRow(uint16(index), field)
+}
+
+func (f *FieldIndexCache) FieldName(index uint16) string {
+	f.mutex.RLock()
+	defer f.mutex.RUnlock()
+	for fieldName, fieldIndex := range f.fieldIndexes {
+		if index == fieldIndex {
+			return fieldName
+		}
+	}
+	return ""
+}

vendor/github.com/blevesearch/bleve/index/upside_down/index_reader.go

@@ -0,0 +1,164 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"bytes"
+
+	"github.com/blevesearch/bleve/document"
+	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/index/store"
+)
+
+type IndexReader struct {
+	index    *UpsideDownCouch
+	kvreader store.KVReader
+	docCount uint64
+}
+
+func (i *IndexReader) TermFieldReader(term []byte, fieldName string) (index.TermFieldReader, error) {
+	fieldIndex, fieldExists := i.index.fieldIndexCache.FieldExists(fieldName)
+	if fieldExists {
+		return newUpsideDownCouchTermFieldReader(i, term, uint16(fieldIndex))
+	}
+	return newUpsideDownCouchTermFieldReader(i, []byte{ByteSeparator}, ^uint16(0))
+}
+
+func (i *IndexReader) FieldDict(fieldName string) (index.FieldDict, error) {
+	return i.FieldDictRange(fieldName, nil, nil)
+}
+
+func (i *IndexReader) FieldDictRange(fieldName string, startTerm []byte, endTerm []byte) (index.FieldDict, error) {
+	fieldIndex, fieldExists := i.index.fieldIndexCache.FieldExists(fieldName)
+	if fieldExists {
+		return newUpsideDownCouchFieldDict(i, uint16(fieldIndex), startTerm, endTerm)
+	}
+	return newUpsideDownCouchFieldDict(i, ^uint16(0), []byte{ByteSeparator}, []byte{})
+}
+
+func (i *IndexReader) FieldDictPrefix(fieldName string, termPrefix []byte) (index.FieldDict, error) {
+	return i.FieldDictRange(fieldName, termPrefix, incrementBytes(termPrefix))
+}
+
+func (i *IndexReader) DocIDReader(start, end string) (index.DocIDReader, error) {
+	return newUpsideDownCouchDocIDReader(i, start, end)
+}
+
+func (i *IndexReader) Document(id string) (doc *document.Document, err error) {
+	// first hit the back index to confirm doc exists
+	var backIndexRow *BackIndexRow
+	backIndexRow, err = i.index.backIndexRowForDoc(i.kvreader, id)
+	if err != nil {
+		return
+	}
+	if backIndexRow == nil {
+		return
+	}
+	doc = document.NewDocument(id)
+	storedRow := NewStoredRow(id, 0, []uint64{}, 'x', nil)
+	storedRowScanPrefix := storedRow.ScanPrefixForDoc()
+	it := i.kvreader.Iterator(storedRowScanPrefix)
+	defer func() {
+		if cerr := it.Close(); err == nil && cerr != nil {
+			err = cerr
+		}
+	}()
+	key, val, valid := it.Current()
+	for valid {
+		if !bytes.HasPrefix(key, storedRowScanPrefix) {
+			break
+		}
+		safeVal := val
+		if !i.kvreader.BytesSafeAfterClose() {
+			safeVal = make([]byte, len(val))
+			copy(safeVal, val)
+		}
+		var row *StoredRow
+		row, err = NewStoredRowKV(key, safeVal)
+		if err != nil {
+			doc = nil
+			return
+		}
+		if row != nil {
+			fieldName := i.index.fieldIndexCache.FieldName(row.field)
+			field := decodeFieldType(row.typ, fieldName, row.value)
+			if field != nil {
+				doc.AddField(field)
+			}
+		}
+
+		it.Next()
+		key, val, valid = it.Current()
+	}
+	return
+}
+
+func (i *IndexReader) DocumentFieldTerms(id string) (index.FieldTerms, error) {
+	back, err := i.index.backIndexRowForDoc(i.kvreader, id)
+	if err != nil {
+		return nil, err
+	}
+	rv := make(index.FieldTerms, len(back.termEntries))
+	for _, entry := range back.termEntries {
+		fieldName := i.index.fieldIndexCache.FieldName(uint16(*entry.Field))
+		terms, ok := rv[fieldName]
+		if !ok {
+			terms = make([]string, 0)
+		}
+		terms = append(terms, *entry.Term)
+		rv[fieldName] = terms
+	}
+	return rv, nil
+}
+
+func (i *IndexReader) Fields() (fields []string, err error) {
+	fields = make([]string, 0)
+	it := i.kvreader.Iterator([]byte{'f'})
+	defer func() {
+		if cerr := it.Close(); err == nil && cerr != nil {
+			err = cerr
+		}
+	}()
+	key, val, valid := it.Current()
+	for valid {
+		if !bytes.HasPrefix(key, []byte{'f'}) {
+			break
+		}
+		var row UpsideDownCouchRow
+		row, err = ParseFromKeyValue(key, val)
+		if err != nil {
+			fields = nil
+			return
+		}
+		if row != nil {
+			fieldRow, ok := row.(*FieldRow)
+			if ok {
+				fields = append(fields, fieldRow.name)
+			}
+		}
+
+		it.Next()
+		key, val, valid = it.Current()
+	}
+	return
+}
+
+func (i *IndexReader) GetInternal(key []byte) ([]byte, error) {
+	internalRow := NewInternalRow(key, nil)
+	return i.kvreader.Get(internalRow.Key())
+}
+
+func (i *IndexReader) DocCount() uint64 {
+	return i.docCount
+}
+
+func (i *IndexReader) Close() error {
+	return i.kvreader.Close()
+}

vendor/github.com/blevesearch/bleve/index/upside_down/reader.go

@@ -0,0 +1,187 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"bytes"
+
+	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/index/store"
+)
+
+type UpsideDownCouchTermFieldReader struct {
+	indexReader  *IndexReader
+	iterator     store.KVIterator
+	count        uint64
+	term         []byte
+	field        uint16
+	readerPrefix []byte
+}
+
+func newUpsideDownCouchTermFieldReader(indexReader *IndexReader, term []byte, field uint16) (*UpsideDownCouchTermFieldReader, error) {
+	dictionaryRow := NewDictionaryRow(term, field, 0)
+	val, err := indexReader.kvreader.Get(dictionaryRow.Key())
+	if err != nil {
+		return nil, err
+	}
+	if val == nil {
+		return &UpsideDownCouchTermFieldReader{
+			count: 0,
+			term:  term,
+			field: field,
+		}, nil
+	}
+
+	err = dictionaryRow.parseDictionaryV(val)
+	if err != nil {
+		return nil, err
+	}
+
+	tfr := NewTermFrequencyRow(term, field, "", 0, 0)
+	readerPrefix := tfr.Key()
+	it := indexReader.kvreader.Iterator(readerPrefix)
+
+	return &UpsideDownCouchTermFieldReader{
+		indexReader:  indexReader,
+		iterator:     it,
+		count:        dictionaryRow.count,
+		term:         term,
+		field:        field,
+		readerPrefix: readerPrefix,
+	}, nil
+}
+
+func (r *UpsideDownCouchTermFieldReader) Count() uint64 {
+	return r.count
+}
+
+func (r *UpsideDownCouchTermFieldReader) Next() (*index.TermFieldDoc, error) {
+	if r.iterator != nil {
+		key, val, valid := r.iterator.Current()
+		if valid {
+			if !bytes.HasPrefix(key, r.readerPrefix) {
+				// end of the line
+				return nil, nil
+			}
+			tfr, err := NewTermFrequencyRowKV(key, val)
+			if err != nil {
+				return nil, err
+			}
+			r.iterator.Next()
+			return &index.TermFieldDoc{
+				ID:      string(tfr.doc),
+				Freq:    tfr.freq,
+				Norm:    float64(tfr.norm),
+				Vectors: r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors),
+			}, nil
+		}
+	}
+	return nil, nil
+}
+
+func (r *UpsideDownCouchTermFieldReader) Advance(docID string) (*index.TermFieldDoc, error) {
+	if r.iterator != nil {
+		tfr := NewTermFrequencyRow(r.term, r.field, docID, 0, 0)
+		r.iterator.Seek(tfr.Key())
+		key, val, valid := r.iterator.Current()
+		if valid {
+			if !bytes.HasPrefix(key, r.readerPrefix) {
+				// end of the line
+				return nil, nil
+			}
+			tfr, err := NewTermFrequencyRowKV(key, val)
+			if err != nil {
+				return nil, err
+			}
+			r.iterator.Next()
+			return &index.TermFieldDoc{
+				ID:      string(tfr.doc),
+				Freq:    tfr.freq,
+				Norm:    float64(tfr.norm),
+				Vectors: r.indexReader.index.termFieldVectorsFromTermVectors(tfr.vectors),
+			}, nil
+		}
+	}
+	return nil, nil
+}
+
+func (r *UpsideDownCouchTermFieldReader) Close() error {
+	if r.iterator != nil {
+		return r.iterator.Close()
+	}
+	return nil
+}
+
+type UpsideDownCouchDocIDReader struct {
+	indexReader *IndexReader
+	iterator    store.KVIterator
+	start       string
+	end         string
+}
+
+func newUpsideDownCouchDocIDReader(indexReader *IndexReader, start, end string) (*UpsideDownCouchDocIDReader, error) {
+	if start == "" {
+		start = string([]byte{0x0})
+	}
+	if end == "" {
+		end = string([]byte{0xff})
+	}
+	bisr := NewBackIndexRow(start, nil, nil)
+	it := indexReader.kvreader.Iterator(bisr.Key())
+
+	return &UpsideDownCouchDocIDReader{
+		indexReader: indexReader,
+		iterator:    it,
+		start:       start,
+		end:         end,
+	}, nil
+}
+
+func (r *UpsideDownCouchDocIDReader) Next() (string, error) {
+	key, val, valid := r.iterator.Current()
+	if valid {
+		bier := NewBackIndexRow(r.end, nil, nil)
+		if bytes.Compare(key, bier.Key()) > 0 {
+			// end of the line
+			return "", nil
+		}
+		br, err := NewBackIndexRowKV(key, val)
+		if err != nil {
+			return "", err
+		}
+		r.iterator.Next()
+		return string(br.doc), nil
+	}
+	return "", nil
+}
+
+func (r *UpsideDownCouchDocIDReader) Advance(docID string) (string, error) {
+	bir := NewBackIndexRow(docID, nil, nil)
+	r.iterator.Seek(bir.Key())
+	key, val, valid := r.iterator.Current()
+	if valid {
+		bier := NewBackIndexRow(r.end, nil, nil)
+		if bytes.Compare(key, bier.Key()) > 0 {
+			// end of the line
+			return "", nil
+		}
+		br, err := NewBackIndexRowKV(key, val)
+		if err != nil {
+			return "", err
+		}
+		r.iterator.Next()
+		return string(br.doc), nil
+	}
+	return "", nil
+}
+
+func (r *UpsideDownCouchDocIDReader) Close() error {
+	return r.iterator.Close()
+}

vendor/github.com/blevesearch/bleve/index/upside_down/reader_test.go

@@ -0,0 +1,297 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/blevesearch/bleve/document"
+	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/index/store/boltdb"
+)
+
+func TestIndexReader(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	s := boltdb.New("test", "bleve")
+	s.SetMergeOperator(&mergeOperator)
+	analysisQueue := NewAnalysisQueue(1)
+	idx := NewUpsideDownCouch(s, analysisQueue)
+	err := idx.Open()
+	if err != nil {
+		t.Errorf("error opening index: %v", err)
+	}
+	defer func() {
+		err := idx.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	var expectedCount uint64
+	doc := document.NewDocument("1")
+	doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+	expectedCount++
+
+	doc = document.NewDocument("2")
+	doc.AddField(document.NewTextFieldWithAnalyzer("name", []uint64{}, []byte("test test test"), testAnalyzer))
+	doc.AddField(document.NewTextFieldCustom("desc", []uint64{}, []byte("eat more rice"), document.IndexField|document.IncludeTermVectors, testAnalyzer))
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+	expectedCount++
+
+	indexReader, err := idx.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := indexReader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	// first look for a term that doesn't exist
+	reader, err := indexReader.TermFieldReader([]byte("nope"), "name")
+	if err != nil {
+		t.Errorf("Error accessing term field reader: %v", err)
+	}
+	count := reader.Count()
+	if count != 0 {
+		t.Errorf("Expected doc count to be: %d got: %d", 0, count)
+	}
+	err = reader.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	reader, err = indexReader.TermFieldReader([]byte("test"), "name")
+	if err != nil {
+		t.Errorf("Error accessing term field reader: %v", err)
+	}
+
+	expectedCount = 2
+	count = reader.Count()
+	if count != expectedCount {
+		t.Errorf("Exptected doc count to be: %d got: %d", expectedCount, count)
+	}
+
+	var match *index.TermFieldDoc
+	var actualCount uint64
+	match, err = reader.Next()
+	for err == nil && match != nil {
+		match, err = reader.Next()
+		if err != nil {
+			t.Errorf("unexpected error reading next")
+		}
+		actualCount++
+	}
+	if actualCount != count {
+		t.Errorf("count was 2, but only saw %d", actualCount)
+	}
+
+	expectedMatch := &index.TermFieldDoc{
+		ID:   "2",
+		Freq: 1,
+		Norm: 0.5773502588272095,
+		Vectors: []*index.TermFieldVector{
+			&index.TermFieldVector{
+				Field: "desc",
+				Pos:   3,
+				Start: 9,
+				End:   13,
+			},
+		},
+	}
+	tfr, err := indexReader.TermFieldReader([]byte("rice"), "desc")
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	match, err = tfr.Next()
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if !reflect.DeepEqual(expectedMatch, match) {
+		t.Errorf("got %#v, expected %#v", match, expectedMatch)
+	}
+	err = reader.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// now test usage of advance
+	reader, err = indexReader.TermFieldReader([]byte("test"), "name")
+	if err != nil {
+		t.Errorf("Error accessing term field reader: %v", err)
+	}
+
+	match, err = reader.Advance("2")
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if match == nil {
+		t.Fatalf("Expected match, got nil")
+	}
+	if match.ID != "2" {
+		t.Errorf("Expected ID '2', got '%s'", match.ID)
+	}
+	match, err = reader.Advance("3")
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if match != nil {
+		t.Errorf("expected nil, got %v", match)
+	}
+	err = reader.Close()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// now test creating a reader for a field that doesn't exist
+	reader, err = indexReader.TermFieldReader([]byte("water"), "doesnotexist")
+	if err != nil {
+		t.Errorf("Error accessing term field reader: %v", err)
+	}
+	count = reader.Count()
+	if count != 0 {
+		t.Errorf("expected count 0 for reader of non-existant field")
+	}
+	match, err = reader.Next()
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if match != nil {
+		t.Errorf("expected nil, got %v", match)
+	}
+	match, err = reader.Advance("anywhere")
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if match != nil {
+		t.Errorf("expected nil, got %v", match)
+	}
+
+}
+
+func TestIndexDocIdReader(t *testing.T) {
+	defer func() {
+		err := os.RemoveAll("test")
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	s := boltdb.New("test", "bleve")
+	s.SetMergeOperator(&mergeOperator)
+	analysisQueue := NewAnalysisQueue(1)
+	idx := NewUpsideDownCouch(s, analysisQueue)
+	err := idx.Open()
+	if err != nil {
+		t.Errorf("error opening index: %v", err)
+	}
+	defer func() {
+		err := idx.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	var expectedCount uint64
+	doc := document.NewDocument("1")
+	doc.AddField(document.NewTextField("name", []uint64{}, []byte("test")))
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+	expectedCount++
+
+	doc = document.NewDocument("2")
+	doc.AddField(document.NewTextField("name", []uint64{}, []byte("test test test")))
+	doc.AddField(document.NewTextFieldWithIndexingOptions("desc", []uint64{}, []byte("eat more rice"), document.IndexField|document.IncludeTermVectors))
+	err = idx.Update(doc)
+	if err != nil {
+		t.Errorf("Error updating index: %v", err)
+	}
+	expectedCount++
+
+	indexReader, err := idx.Reader()
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		err := indexReader.Close()
+		if err != nil {
+			t.Error(err)
+		}
+	}()
+
+	// first get all doc ids
+	reader, err := indexReader.DocIDReader("", "")
+	if err != nil {
+		t.Errorf("Error accessing doc id reader: %v", err)
+	}
+	defer func() {
+		err := reader.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}()
+
+	id, err := reader.Next()
+	count := uint64(0)
+	for id != "" {
+		count++
+		id, err = reader.Next()
+	}
+	if count != expectedCount {
+		t.Errorf("expected %d, got %d", expectedCount, count)
+	}
+
+	// try it again, but jump to the second doc this time
+	reader2, err := indexReader.DocIDReader("", "")
+	if err != nil {
+		t.Errorf("Error accessing doc id reader: %v", err)
+	}
+	defer func() {
+		err := reader2.Close()
+		if err != nil {
+			t.Error(err)
+		}
+	}()
+
+	id, err = reader2.Advance("2")
+	if err != nil {
+		t.Error(err)
+	}
+	if id != "2" {
+		t.Errorf("expected to find id '2', got '%s'", id)
+	}
+
+	id, err = reader2.Advance("3")
+	if err != nil {
+		t.Error(err)
+	}
+	if id != "" {
+		t.Errorf("expected to find id '', got '%s'", id)
+	}
+}

vendor/github.com/blevesearch/bleve/index/upside_down/row.go

@@ -0,0 +1,641 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"io"
+	"math"
+
+	"github.com/golang/protobuf/proto"
+)
+
+const ByteSeparator byte = 0xff
+
+type UpsideDownCouchRowStream chan UpsideDownCouchRow
+
+type UpsideDownCouchRow interface {
+	Key() []byte
+	Value() []byte
+}
+
+func ParseFromKeyValue(key, value []byte) (UpsideDownCouchRow, error) {
+	if len(key) > 0 {
+		switch key[0] {
+		case 'v':
+			return NewVersionRowKV(key, value)
+		case 'f':
+			return NewFieldRowKV(key, value)
+		case 'd':
+			return NewDictionaryRowKV(key, value)
+		case 't':
+			return NewTermFrequencyRowKV(key, value)
+		case 'b':
+			return NewBackIndexRowKV(key, value)
+		case 's':
+			return NewStoredRowKV(key, value)
+		case 'i':
+			return NewInternalRowKV(key, value)
+		}
+		return nil, fmt.Errorf("Unknown field type '%s'", string(key[0]))
+	}
+	return nil, fmt.Errorf("Invalid empty key")
+}
+
+// VERSION
+
+type VersionRow struct {
+	version uint8
+}
+
+func (v *VersionRow) Key() []byte {
+	return []byte{'v'}
+}
+
+func (v *VersionRow) Value() []byte {
+	return []byte{byte(v.version)}
+}
+
+func (v *VersionRow) String() string {
+	return fmt.Sprintf("Version: %d", v.version)
+}
+
+func NewVersionRow(version uint8) *VersionRow {
+	return &VersionRow{
+		version: version,
+	}
+}
+
+func NewVersionRowKV(key, value []byte) (*VersionRow, error) {
+	rv := VersionRow{}
+	buf := bytes.NewBuffer(value)
+	err := binary.Read(buf, binary.LittleEndian, &rv.version)
+	if err != nil {
+		return nil, err
+	}
+	return &rv, nil
+}
+
+// INTERNAL STORAGE
+
+type InternalRow struct {
+	key []byte
+	val []byte
+}
+
+func (i *InternalRow) Key() []byte {
+	buf := make([]byte, len(i.key)+1)
+	buf[0] = 'i'
+	copy(buf[1:], i.key)
+	return buf
+}
+
+func (i *InternalRow) Value() []byte {
+	return i.val
+}
+
+func (i *InternalRow) String() string {
+	return fmt.Sprintf("InternalStore - Key: %s (% x) Val: %s (% x)", i.key, i.key, i.val, i.val)
+}
+
+func NewInternalRow(key, val []byte) *InternalRow {
+	return &InternalRow{
+		key: key,
+		val: val,
+	}
+}
+
+func NewInternalRowKV(key, value []byte) (*InternalRow, error) {
+	rv := InternalRow{}
+	rv.key = key[1:]
+	rv.val = value
+	return &rv, nil
+}
+
+// FIELD definition
+
+type FieldRow struct {
+	index uint16
+	name  string
+}
+
+func (f *FieldRow) Key() []byte {
+	buf := make([]byte, 3)
+	buf[0] = 'f'
+	binary.LittleEndian.PutUint16(buf[1:3], f.index)
+	return buf
+}
+
+func (f *FieldRow) Value() []byte {
+	return append([]byte(f.name), ByteSeparator)
+}
+
+func (f *FieldRow) String() string {
+	return fmt.Sprintf("Field: %d Name: %s", f.index, f.name)
+}
+
+func NewFieldRow(index uint16, name string) *FieldRow {
+	return &FieldRow{
+		index: index,
+		name:  name,
+	}
+}
+
+func NewFieldRowKV(key, value []byte) (*FieldRow, error) {
+	rv := FieldRow{}
+
+	buf := bytes.NewBuffer(key)
+	_, err := buf.ReadByte() // type
+	if err != nil {
+		return nil, err
+	}
+	err = binary.Read(buf, binary.LittleEndian, &rv.index)
+	if err != nil {
+		return nil, err
+	}
+
+	buf = bytes.NewBuffer(value)
+	rv.name, err = buf.ReadString(ByteSeparator)
+	if err != nil {
+		return nil, err
+	}
+	rv.name = rv.name[:len(rv.name)-1] // trim off separator byte
+
+	return &rv, nil
+}
+
+// DICTIONARY
+
+type DictionaryRow struct {
+	field uint16
+	term  []byte
+	count uint64
+}
+
+func (dr *DictionaryRow) Key() []byte {
+	buf := make([]byte, 3+len(dr.term))
+	buf[0] = 'd'
+	binary.LittleEndian.PutUint16(buf[1:3], dr.field)
+	copy(buf[3:], dr.term)
+	return buf
+}
+
+func (dr *DictionaryRow) Value() []byte {
+	used := 0
+	buf := make([]byte, 8)
+
+	used += binary.PutUvarint(buf[used:used+8], dr.count)
+
+	return buf[0:used]
+}
+
+func (dr *DictionaryRow) String() string {
+	return fmt.Sprintf("Dictionary Term: `%s` Field: %d Count: %d ", string(dr.term), dr.field, dr.count)
+}
+
+func NewDictionaryRow(term []byte, field uint16, count uint64) *DictionaryRow {
+	return &DictionaryRow{
+		term:  term,
+		field: field,
+		count: count,
+	}
+}
+
+func NewDictionaryRowKV(key, value []byte) (*DictionaryRow, error) {
+	rv, err := NewDictionaryRowK(key)
+	if err != nil {
+		return nil, err
+	}
+
+	err = rv.parseDictionaryV(value)
+	if err != nil {
+		return nil, err
+	}
+	return rv, nil
+
+}
+
+func NewDictionaryRowK(key []byte) (*DictionaryRow, error) {
+	rv := DictionaryRow{}
+	buf := bytes.NewBuffer(key)
+	_, err := buf.ReadByte() // type
+	if err != nil {
+		return nil, err
+	}
+
+	err = binary.Read(buf, binary.LittleEndian, &rv.field)
+	if err != nil {
+		return nil, err
+	}
+
+	rv.term, err = buf.ReadBytes(ByteSeparator)
+	// there is no separator expected here, should get EOF
+	if err != io.EOF {
+		return nil, err
+	}
+
+	return &rv, nil
+}
+
+func (dr *DictionaryRow) parseDictionaryV(value []byte) error {
+	buf := bytes.NewBuffer((value))
+
+	count, err := binary.ReadUvarint(buf)
+	if err != nil {
+		return err
+	}
+	dr.count = count
+
+	return nil
+}
+
+// TERM FIELD FREQUENCY
+
+type TermVector struct {
+	field uint16
+	pos   uint64
+	start uint64
+	end   uint64
+}
+
+func (tv *TermVector) String() string {
+	return fmt.Sprintf("Field: %d Pos: %d Start: %d End %d", tv.field, tv.pos, tv.start, tv.end)
+}
+
+type TermFrequencyRow struct {
+	term    []byte
+	field   uint16
+	doc     []byte
+	freq    uint64
+	norm    float32
+	vectors []*TermVector
+}
+
+func (tfr *TermFrequencyRow) ScanPrefixForField() []byte {
+	buf := make([]byte, 3)
+	buf[0] = 't'
+	binary.LittleEndian.PutUint16(buf[1:3], tfr.field)
+	return buf
+}
+
+func (tfr *TermFrequencyRow) ScanPrefixForFieldTermPrefix() []byte {
+	buf := make([]byte, 3+len(tfr.term))
+	buf[0] = 't'
+	binary.LittleEndian.PutUint16(buf[1:3], tfr.field)
+	copy(buf[3:], tfr.term)
+	return buf
+}
+
+func (tfr *TermFrequencyRow) ScanPrefixForFieldTerm() []byte {
+	buf := make([]byte, 3+len(tfr.term)+1)
+	buf[0] = 't'
+	binary.LittleEndian.PutUint16(buf[1:3], tfr.field)
+	termLen := copy(buf[3:], tfr.term)
+	buf[3+termLen] = ByteSeparator
+	return buf
+}
+
+func (tfr *TermFrequencyRow) Key() []byte {
+	buf := make([]byte, 3+len(tfr.term)+1+len(tfr.doc))
+	buf[0] = 't'
+	binary.LittleEndian.PutUint16(buf[1:3], tfr.field)
+	termLen := copy(buf[3:], tfr.term)
+	buf[3+termLen] = ByteSeparator
+	copy(buf[3+termLen+1:], tfr.doc)
+	return buf
+}
+
+func (tfr *TermFrequencyRow) DictionaryRowKey() []byte {
+	dr := NewDictionaryRow(tfr.term, tfr.field, 0)
+	return dr.Key()
+}
+
+func (tfr *TermFrequencyRow) Value() []byte {
+	used := 0
+	buf := make([]byte, 8+8+(len(tfr.vectors)*(8+8+8+8)))
+
+	used += binary.PutUvarint(buf[used:used+8], tfr.freq)
+
+	normuint32 := math.Float32bits(tfr.norm)
+	newbuf := buf[used : used+8]
+	used += binary.PutUvarint(newbuf, uint64(normuint32))
+
+	for _, vector := range tfr.vectors {
+		used += binary.PutUvarint(buf[used:used+8], uint64(vector.field))
+		used += binary.PutUvarint(buf[used:used+8], vector.pos)
+		used += binary.PutUvarint(buf[used:used+8], vector.start)
+		used += binary.PutUvarint(buf[used:used+8], vector.end)
+	}
+	return buf[0:used]
+}
+
+func (tfr *TermFrequencyRow) String() string {
+	return fmt.Sprintf("Term: `%s` Field: %d DocId: `%s` Frequency: %d Norm: %f Vectors: %v", string(tfr.term), tfr.field, string(tfr.doc), tfr.freq, tfr.norm, tfr.vectors)
+}
+
+func NewTermFrequencyRow(term []byte, field uint16, doc string, freq uint64, norm float32) *TermFrequencyRow {
+	return &TermFrequencyRow{
+		term:  term,
+		field: field,
+		doc:   []byte(doc),
+		freq:  freq,
+		norm:  norm,
+	}
+}
+
+func NewTermFrequencyRowWithTermVectors(term []byte, field uint16, doc string, freq uint64, norm float32, vectors []*TermVector) *TermFrequencyRow {
+	return &TermFrequencyRow{
+		term:    term,
+		field:   field,
+		doc:     []byte(doc),
+		freq:    freq,
+		norm:    norm,
+		vectors: vectors,
+	}
+}
+
+func NewTermFrequencyRowK(key []byte) (*TermFrequencyRow, error) {
+	rv := TermFrequencyRow{}
+	keyLen := len(key)
+	if keyLen < 3 {
+		return nil, fmt.Errorf("invalid term frequency key, no valid field")
+	}
+	rv.field = binary.LittleEndian.Uint16(key[1:3])
+
+	termEndPos := bytes.IndexByte(key[3:], ByteSeparator)
+	if termEndPos < 0 {
+		return nil, fmt.Errorf("invalid term frequency key, no byte separator terminating term")
+	}
+	rv.term = key[3 : 3+termEndPos]
+
+	docLen := len(key) - (3 + termEndPos + 1)
+	if docLen < 1 {
+		return nil, fmt.Errorf("invalid term frequency key, empty docid")
+	}
+	rv.doc = key[3+termEndPos+1:]
+
+	return &rv, nil
+}
+
+func (tfr *TermFrequencyRow) parseV(value []byte) error {
+	currOffset := 0
+	bytesRead := 0
+	tfr.freq, bytesRead = binary.Uvarint(value[currOffset:])
+	if bytesRead <= 0 {
+		return fmt.Errorf("invalid term frequency value, invalid frequency")
+	}
+	currOffset += bytesRead
+
+	var norm uint64
+	norm, bytesRead = binary.Uvarint(value[currOffset:])
+	if bytesRead <= 0 {
+		return fmt.Errorf("invalid term frequency value, no norm")
+	}
+	currOffset += bytesRead
+
+	tfr.norm = math.Float32frombits(uint32(norm))
+
+	var field uint64
+	field, bytesRead = binary.Uvarint(value[currOffset:])
+	for bytesRead > 0 {
+		currOffset += bytesRead
+		tv := TermVector{}
+		tv.field = uint16(field)
+		// at this point we expect at least one term vector
+		if tfr.vectors == nil {
+			tfr.vectors = make([]*TermVector, 0)
+		}
+
+		tv.pos, bytesRead = binary.Uvarint(value[currOffset:])
+		if bytesRead <= 0 {
+			return fmt.Errorf("invalid term frequency value, vector contains no position")
+		}
+		currOffset += bytesRead
+
+		tv.start, bytesRead = binary.Uvarint(value[currOffset:])
+		if bytesRead <= 0 {
+			return fmt.Errorf("invalid term frequency value, vector contains no start")
+		}
+		currOffset += bytesRead
+
+		tv.end, bytesRead = binary.Uvarint(value[currOffset:])
+		if bytesRead <= 0 {
+			return fmt.Errorf("invalid term frequency value, vector contains no end")
+		}
+		currOffset += bytesRead
+
+		tfr.vectors = append(tfr.vectors, &tv)
+		// try to read next record (may not exist)
+		field, bytesRead = binary.Uvarint(value[currOffset:])
+	}
+	if len(value[currOffset:]) > 0 && bytesRead <= 0 {
+		return fmt.Errorf("invalid term frequency value, vector field invalid")
+	}
+
+	return nil
+}
+
+func NewTermFrequencyRowKV(key, value []byte) (*TermFrequencyRow, error) {
+	rv, err := NewTermFrequencyRowK(key)
+	if err != nil {
+		return nil, err
+	}
+
+	err = rv.parseV(value)
+	if err != nil {
+		return nil, err
+	}
+	return rv, nil
+
+}
+
+type BackIndexRow struct {
+	doc           []byte
+	termEntries   []*BackIndexTermEntry
+	storedEntries []*BackIndexStoreEntry
+}
+
+func (br *BackIndexRow) AllTermKeys() [][]byte {
+	if br == nil {
+		return nil
+	}
+	rv := make([][]byte, len(br.termEntries))
+	for i, termEntry := range br.termEntries {
+		termRow := NewTermFrequencyRow([]byte(termEntry.GetTerm()), uint16(termEntry.GetField()), string(br.doc), 0, 0)
+		rv[i] = termRow.Key()
+	}
+	return rv
+}
+
+func (br *BackIndexRow) AllStoredKeys() [][]byte {
+	if br == nil {
+		return nil
+	}
+	rv := make([][]byte, len(br.storedEntries))
+	for i, storedEntry := range br.storedEntries {
+		storedRow := NewStoredRow(string(br.doc), uint16(storedEntry.GetField()), storedEntry.GetArrayPositions(), 'x', []byte{})
+		rv[i] = storedRow.Key()
+	}
+	return rv
+}
+
+func (br *BackIndexRow) Key() []byte {
+	buf := make([]byte, len(br.doc)+1)
+	buf[0] = 'b'
+	copy(buf[1:], br.doc)
+	return buf
+}
+
+func (br *BackIndexRow) Value() []byte {
+	birv := &BackIndexRowValue{
+		TermEntries:   br.termEntries,
+		StoredEntries: br.storedEntries,
+	}
+	bytes, _ := proto.Marshal(birv)
+	return bytes
+}
+
+func (br *BackIndexRow) String() string {
+	return fmt.Sprintf("Backindex DocId: `%s` Term Entries: %v, Stored Entries: %v", string(br.doc), br.termEntries, br.storedEntries)
+}
+
+func NewBackIndexRow(doc string, entries []*BackIndexTermEntry, storedFields []*BackIndexStoreEntry) *BackIndexRow {
+	return &BackIndexRow{
+		doc:           []byte(doc),
+		termEntries:   entries,
+		storedEntries: storedFields,
+	}
+}
+
+func NewBackIndexRowKV(key, value []byte) (*BackIndexRow, error) {
+	rv := BackIndexRow{}
+
+	buf := bytes.NewBuffer(key)
+	_, err := buf.ReadByte() // type
+	if err != nil {
+		return nil, err
+	}
+
+	rv.doc, err = buf.ReadBytes(ByteSeparator)
+	if err == io.EOF && len(rv.doc) < 1 {
+		err = fmt.Errorf("invalid doc length 0")
+	}
+	if err != io.EOF {
+		return nil, err
+	}
+
+	var birv BackIndexRowValue
+	err = proto.Unmarshal(value, &birv)
+	if err != nil {
+		return nil, err
+	}
+	rv.termEntries = birv.TermEntries
+	rv.storedEntries = birv.StoredEntries
+
+	return &rv, nil
+}
+
+// STORED
+
+type StoredRow struct {
+	doc            []byte
+	field          uint16
+	arrayPositions []uint64
+	typ            byte
+	value          []byte
+}
+
+func (s *StoredRow) Key() []byte {
+	docLen := len(s.doc)
+	buf := make([]byte, 1+docLen+1+2+(binary.MaxVarintLen64*len(s.arrayPositions)))
+	buf[0] = 's'
+	copy(buf[1:], s.doc)
+	buf[1+docLen] = ByteSeparator
+	binary.LittleEndian.PutUint16(buf[1+docLen+1:], s.field)
+	bytesUsed := 1 + docLen + 1 + 2
+	for _, arrayPosition := range s.arrayPositions {
+		varbytes := binary.PutUvarint(buf[bytesUsed:], arrayPosition)
+		bytesUsed += varbytes
+	}
+	return buf[0:bytesUsed]
+}
+
+func (s *StoredRow) Value() []byte {
+	rv := make([]byte, len(s.value)+1)
+	rv[0] = s.typ
+	copy(rv[1:], s.value)
+	return rv
+}
+
+func (s *StoredRow) String() string {
+	return fmt.Sprintf("Document: %s Field %d, Array Positions: %v, Type: %s Value: %s", s.doc, s.field, s.arrayPositions, string(s.typ), s.value)
+}
+
+func (s *StoredRow) ScanPrefixForDoc() []byte {
+	docLen := len(s.doc)
+	buf := make([]byte, 1+docLen+1)
+	buf[0] = 's'
+	copy(buf[1:], s.doc)
+	buf[1+docLen] = ByteSeparator
+	return buf
+}
+
+func NewStoredRow(doc string, field uint16, arrayPositions []uint64, typ byte, value []byte) *StoredRow {
+	return &StoredRow{
+		doc:            []byte(doc),
+		field:          field,
+		arrayPositions: arrayPositions,
+		typ:            typ,
+		value:          value,
+	}
+}
+
+func NewStoredRowK(key []byte) (*StoredRow, error) {
+	rv := StoredRow{}
+
+	buf := bytes.NewBuffer(key)
+	_, err := buf.ReadByte() // type
+	if err != nil {
+		return nil, err
+	}
+
+	rv.doc, err = buf.ReadBytes(ByteSeparator)
+	if len(rv.doc) < 2 { // 1 for min doc id length, 1 for separator
+		err = fmt.Errorf("invalid doc length 0")
+		return nil, err
+	}
+
+	rv.doc = rv.doc[:len(rv.doc)-1] // trim off separator byte
+
+	err = binary.Read(buf, binary.LittleEndian, &rv.field)
+	if err != nil {
+		return nil, err
+	}
+
+	rv.arrayPositions = make([]uint64, 0)
+	nextArrayPos, err := binary.ReadUvarint(buf)
+	for err == nil {
+		rv.arrayPositions = append(rv.arrayPositions, nextArrayPos)
+		nextArrayPos, err = binary.ReadUvarint(buf)
+	}
+	return &rv, nil
+}
+
+func NewStoredRowKV(key, value []byte) (*StoredRow, error) {
+	rv, err := NewStoredRowK(key)
+	if err != nil {
+		return nil, err
+	}
+	rv.typ = value[0]
+	rv.value = value[1:]
+	return rv, nil
+}

vendor/github.com/blevesearch/bleve/index/upside_down/row_merge.go

@@ -0,0 +1,70 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"encoding/binary"
+)
+
+var mergeOperator upsideDownMerge
+
+var dictionaryTermIncr []byte
+var dictionaryTermDecr []byte
+
+func init() {
+	dictionaryTermIncr = make([]byte, 8)
+	binary.LittleEndian.PutUint64(dictionaryTermIncr, uint64(1))
+	dictionaryTermDecr = make([]byte, 8)
+	var negOne = int64(-1)
+	binary.LittleEndian.PutUint64(dictionaryTermDecr, uint64(negOne))
+}
+
+type upsideDownMerge struct{}
+
+func (m *upsideDownMerge) FullMerge(key, existingValue []byte, operands [][]byte) ([]byte, bool) {
+	// set up record based on key
+	dr, err := NewDictionaryRowK(key)
+	if err != nil {
+		return nil, false
+	}
+	if len(existingValue) > 0 {
+		// if existing value, parse it
+		err = dr.parseDictionaryV(existingValue)
+		if err != nil {
+			return nil, false
+		}
+	}
+
+	// now process operands
+	for _, operand := range operands {
+		next := int64(binary.LittleEndian.Uint64(operand))
+		if next < 0 && uint64(-next) > dr.count {
+			// subtracting next from existing would overflow
+			dr.count = 0
+		} else if next < 0 {
+			dr.count -= uint64(-next)
+		} else {
+			dr.count += uint64(next)
+		}
+	}
+
+	return dr.Value(), true
+}
+
+func (m *upsideDownMerge) PartialMerge(key, leftOperand, rightOperand []byte) ([]byte, bool) {
+	left := int64(binary.LittleEndian.Uint64(leftOperand))
+	right := int64(binary.LittleEndian.Uint64(rightOperand))
+	binary.LittleEndian.PutUint64(leftOperand, uint64(left+right))
+	return leftOperand, true
+}
+
+func (m *upsideDownMerge) Name() string {
+	return "upsideDownMerge"
+}

vendor/github.com/blevesearch/bleve/index/upside_down/row_test.go

@@ -0,0 +1,329 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/golang/protobuf/proto"
+)
+
+func TestRows(t *testing.T) {
+	tests := []struct {
+		input  UpsideDownCouchRow
+		outKey []byte
+		outVal []byte
+	}{
+		{
+			NewVersionRow(1),
+			[]byte{'v'},
+			[]byte{0x1},
+		},
+		{
+			NewFieldRow(0, "name"),
+			[]byte{'f', 0, 0},
+			[]byte{'n', 'a', 'm', 'e', ByteSeparator},
+		},
+		{
+			NewFieldRow(1, "desc"),
+			[]byte{'f', 1, 0},
+			[]byte{'d', 'e', 's', 'c', ByteSeparator},
+		},
+		{
+			NewFieldRow(513, "style"),
+			[]byte{'f', 1, 2},
+			[]byte{'s', 't', 'y', 'l', 'e', ByteSeparator},
+		},
+		{
+			NewDictionaryRow([]byte{'b', 'e', 'e', 'r'}, 0, 27),
+			[]byte{'d', 0, 0, 'b', 'e', 'e', 'r'},
+			[]byte{27},
+		},
+		{
+			NewTermFrequencyRow([]byte{'b', 'e', 'e', 'r'}, 0, "catz", 3, 3.14),
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'c', 'a', 't', 'z'},
+			[]byte{3, 195, 235, 163, 130, 4},
+		},
+		{
+			NewTermFrequencyRow([]byte{'b', 'e', 'e', 'r'}, 0, "budweiser", 3, 3.14),
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{3, 195, 235, 163, 130, 4},
+		},
+		{
+			NewTermFrequencyRowWithTermVectors([]byte{'b', 'e', 'e', 'r'}, 0, "budweiser", 3, 3.14, []*TermVector{&TermVector{field: 0, pos: 1, start: 3, end: 11}, &TermVector{field: 0, pos: 2, start: 23, end: 31}, &TermVector{field: 0, pos: 3, start: 43, end: 51}}),
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{3, 195, 235, 163, 130, 4, 0, 1, 3, 11, 0, 2, 23, 31, 0, 3, 43, 51},
+		},
+		// test larger varints
+		{
+			NewTermFrequencyRowWithTermVectors([]byte{'b', 'e', 'e', 'r'}, 0, "budweiser", 25896, 3.14, []*TermVector{&TermVector{field: 255, pos: 1, start: 3, end: 11}, &TermVector{field: 0, pos: 2198, start: 23, end: 31}, &TermVector{field: 0, pos: 3, start: 43, end: 51}}),
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{168, 202, 1, 195, 235, 163, 130, 4, 255, 1, 1, 3, 11, 0, 150, 17, 23, 31, 0, 3, 43, 51},
+		},
+		{
+			NewBackIndexRow("budweiser", []*BackIndexTermEntry{&BackIndexTermEntry{Term: proto.String("beer"), Field: proto.Uint32(0)}}, nil),
+			[]byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{10, 8, 10, 4, 'b', 'e', 'e', 'r', 16, 0},
+		},
+		{
+			NewBackIndexRow("budweiser", []*BackIndexTermEntry{&BackIndexTermEntry{Term: proto.String("beer"), Field: proto.Uint32(0)}, &BackIndexTermEntry{Term: proto.String("beat"), Field: proto.Uint32(1)}}, nil),
+			[]byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{10, 8, 10, 4, 'b', 'e', 'e', 'r', 16, 0, 10, 8, 10, 4, 'b', 'e', 'a', 't', 16, 1},
+		},
+		{
+			NewBackIndexRow("budweiser", []*BackIndexTermEntry{&BackIndexTermEntry{Term: proto.String("beer"), Field: proto.Uint32(0)}, &BackIndexTermEntry{Term: proto.String("beat"), Field: proto.Uint32(1)}}, []*BackIndexStoreEntry{&BackIndexStoreEntry{Field: proto.Uint32(3)}, &BackIndexStoreEntry{Field: proto.Uint32(4)}, &BackIndexStoreEntry{Field: proto.Uint32(5)}}),
+			[]byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{10, 8, 10, 4, 'b', 'e', 'e', 'r', 16, 0, 10, 8, 10, 4, 'b', 'e', 'a', 't', 16, 1, 18, 2, 8, 3, 18, 2, 8, 4, 18, 2, 8, 5},
+		},
+		{
+			NewStoredRow("budweiser", 0, []uint64{}, byte('t'), []byte("an american beer")),
+			[]byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator, 0, 0},
+			[]byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'},
+		},
+		{
+			NewStoredRow("budweiser", 0, []uint64{2, 294, 3078}, byte('t'), []byte("an american beer")),
+			[]byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator, 0, 0, 2, 166, 2, 134, 24},
+			[]byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'},
+		},
+		{
+			NewInternalRow([]byte("mapping"), []byte(`{"mapping":"json content"}`)),
+			[]byte{'i', 'm', 'a', 'p', 'p', 'i', 'n', 'g'},
+			[]byte{'{', '"', 'm', 'a', 'p', 'p', 'i', 'n', 'g', '"', ':', '"', 'j', 's', 'o', 'n', ' ', 'c', 'o', 'n', 't', 'e', 'n', 't', '"', '}'},
+		},
+	}
+
+	// test going from struct to k/v bytes
+	for i, test := range tests {
+		rk := test.input.Key()
+		if !reflect.DeepEqual(rk, test.outKey) {
+			t.Errorf("Expected key to be %v got: %v", test.outKey, rk)
+		}
+		rv := test.input.Value()
+		if !reflect.DeepEqual(rv, test.outVal) {
+			t.Errorf("Expected value to be %v got: %v for %d", test.outVal, rv, i)
+		}
+	}
+
+	// now test going back from k/v bytes to struct
+	for i, test := range tests {
+		row, err := ParseFromKeyValue(test.outKey, test.outVal)
+		if err != nil {
+			t.Errorf("error parsking key/value: %v", err)
+		}
+		if !reflect.DeepEqual(row, test.input) {
+			t.Errorf("Expected: %#v got: %#v for %d", test.input, row, i)
+		}
+	}
+
+}
+
+func TestInvalidRows(t *testing.T) {
+	tests := []struct {
+		key []byte
+		val []byte
+	}{
+		// empty key
+		{
+			[]byte{},
+			[]byte{},
+		},
+		// no such type q
+		{
+			[]byte{'q'},
+			[]byte{},
+		},
+		// type v, invalid empty value
+		{
+			[]byte{'v'},
+			[]byte{},
+		},
+		// type f, invalid key
+		{
+			[]byte{'f'},
+			[]byte{},
+		},
+		// type f, valid key, invalid value
+		{
+			[]byte{'f', 0, 0},
+			[]byte{},
+		},
+		// type t, invalid key (missing field)
+		{
+			[]byte{'t'},
+			[]byte{},
+		},
+		// type t, invalid key (missing term)
+		{
+			[]byte{'t', 0, 0},
+			[]byte{},
+		},
+		// type t, invalid key (missing id)
+		{
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator},
+			[]byte{},
+		},
+		// type t, invalid val (missing freq)
+		{
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{},
+		},
+		// type t, invalid val (missing norm)
+		{
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{3},
+		},
+		// type t, invalid val (half missing tv field, full missing is valid (no term vectors))
+		{
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{3, 25, 255},
+		},
+		// type t, invalid val (missing tv pos)
+		{
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{3, 25, 0},
+		},
+		// type t, invalid val (missing tv start)
+		{
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{3, 25, 0, 0},
+		},
+		// type t, invalid val (missing tv end)
+		{
+			[]byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{3, 25, 0, 0, 0},
+		},
+		// type b, invalid key (missing id)
+		{
+			[]byte{'b'},
+			[]byte{'b', 'e', 'e', 'r', ByteSeparator, 0, 0},
+		},
+		// type b, invalid val (missing field)
+		{
+			[]byte{'b', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'},
+			[]byte{'g', 'a', 'r', 'b', 'a', 'g', 'e'},
+		},
+		// type s, invalid key (missing id)
+		{
+			[]byte{'s'},
+			[]byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'},
+		},
+		// type b, invalid val (missing field)
+		{
+			[]byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator},
+			[]byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'},
+		},
+	}
+
+	for _, test := range tests {
+		_, err := ParseFromKeyValue(test.key, test.val)
+		if err == nil {
+			t.Errorf("expected error, got nil")
+		}
+	}
+}
+
+func BenchmarkTermFrequencyRowEncode(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		row := NewTermFrequencyRowWithTermVectors(
+			[]byte{'b', 'e', 'e', 'r'},
+			0,
+			"budweiser",
+			3,
+			3.14,
+			[]*TermVector{
+				&TermVector{
+					field: 0,
+					pos:   1,
+					start: 3,
+					end:   11,
+				},
+				&TermVector{
+					field: 0,
+					pos:   2,
+					start: 23,
+					end:   31,
+				},
+				&TermVector{
+					field: 0,
+					pos:   3,
+					start: 43,
+					end:   51,
+				},
+			})
+
+		row.Key()
+		row.Value()
+	}
+}
+
+func BenchmarkTermFrequencyRowDecode(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		k := []byte{'t', 0, 0, 'b', 'e', 'e', 'r', ByteSeparator, 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r'}
+		v := []byte{3, 195, 235, 163, 130, 4, 0, 1, 3, 11, 0, 2, 23, 31, 0, 3, 43, 51}
+		_, err := NewTermFrequencyRowKV(k, v)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkBackIndexRowEncode(b *testing.B) {
+	field := uint32(1)
+	t1 := "term1"
+	for i := 0; i < b.N; i++ {
+		row := NewBackIndexRow("beername",
+			[]*BackIndexTermEntry{
+				&BackIndexTermEntry{
+					Term:  &t1,
+					Field: &field,
+				},
+			},
+			[]*BackIndexStoreEntry{
+				&BackIndexStoreEntry{
+					Field: &field,
+				},
+			})
+
+		row.Key()
+		row.Value()
+	}
+}
+
+func BenchmarkBackIndexRowDecode(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		k := []byte{0x62, 0x62, 0x65, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65}
+		v := []byte{0x0a, 0x09, 0x0a, 0x05, 0x74, 0x65, 0x72, 0x6d, 0x31, 0x10, 0x01, 0x12, 0x02, 0x08, 0x01}
+		_, err := NewBackIndexRowKV(k, v)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkStoredRowEncode(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		row := NewStoredRow("budweiser", 0, []uint64{}, byte('t'), []byte("an american beer"))
+
+		row.Key()
+		row.Value()
+	}
+}
+
+func BenchmarkStoredRowDecode(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		k := []byte{'s', 'b', 'u', 'd', 'w', 'e', 'i', 's', 'e', 'r', ByteSeparator, 0, 0}
+		v := []byte{'t', 'a', 'n', ' ', 'a', 'm', 'e', 'r', 'i', 'c', 'a', 'n', ' ', 'b', 'e', 'e', 'r'}
+		_, err := NewStoredRowKV(k, v)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

vendor/github.com/blevesearch/bleve/index/upside_down/stats.go

@@ -0,0 +1,31 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"encoding/json"
+	"sync/atomic"
+)
+
+type indexStat struct {
+	updates, deletes, batches, errors uint64
+	analysisTime, indexTime           uint64
+}
+
+func (i *indexStat) MarshalJSON() ([]byte, error) {
+	m := map[string]interface{}{}
+	m["updates"] = atomic.LoadUint64(&i.updates)
+	m["deletes"] = atomic.LoadUint64(&i.deletes)
+	m["batches"] = atomic.LoadUint64(&i.batches)
+	m["errors"] = atomic.LoadUint64(&i.errors)
+	m["analysis_time"] = atomic.LoadUint64(&i.analysisTime)
+	m["index_time"] = atomic.LoadUint64(&i.indexTime)
+	return json.Marshal(m)
+}

vendor/github.com/blevesearch/bleve/index/upside_down/upside_down.go

@@ -0,0 +1,721 @@
+//  Copyright (c) 2014 Couchbase, Inc.
+//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+//  except in compliance with the License. You may obtain a copy of the License at
+//    http://www.apache.org/licenses/LICENSE-2.0
+//  Unless required by applicable law or agreed to in writing, software distributed under the
+//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+//  either express or implied. See the License for the specific language governing permissions
+//  and limitations under the License.
+
+package upside_down
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"math"
+	"sync/atomic"
+	"time"
+
+	"github.com/blevesearch/bleve/analysis"
+	"github.com/blevesearch/bleve/document"
+	"github.com/blevesearch/bleve/index"
+	"github.com/blevesearch/bleve/index/store"
+
+	"github.com/golang/protobuf/proto"
+)
+
+var VersionKey = []byte{'v'}
+
+const Version uint8 = 4
+
+var IncompatibleVersion = fmt.Errorf("incompatible version, %d is supported", Version)
+
+type UpsideDownCouch struct {
+	version         uint8
+	path            string
+	store           store.KVStore
+	fieldIndexCache *FieldIndexCache
+	docCount        uint64
+	analysisQueue   *AnalysisQueue
+	stats           *indexStat
+}
+
+func NewUpsideDownCouch(s store.KVStore, analysisQueue *AnalysisQueue) *UpsideDownCouch {
+	return &UpsideDownCouch{
+		version:         Version,
+		fieldIndexCache: NewFieldIndexCache(),
+		store:           s,
+		analysisQueue:   analysisQueue,
+		stats:           &indexStat{},
+	}
+}
+
+func (udc *UpsideDownCouch) init(kvwriter store.KVWriter) (err error) {
+	// prepare a list of rows
+	rows := make([]UpsideDownCouchRow, 0)
+
+	// version marker
+	rows = append(rows, NewVersionRow(udc.version))
+
+	return udc.batchRows(kvwriter, nil, rows, nil)
+}
+
+func (udc *UpsideDownCouch) loadSchema(kvreader store.KVReader) (err error) {
+
+	keyPrefix := []byte{'f'}
+	it := kvreader.Iterator(keyPrefix)
+	defer func() {
+		if cerr := it.Close(); err == nil && cerr != nil {
+			err = cerr
+		}
+	}()
+
+	it.Seek(keyPrefix)
+	key, val, valid := it.Current()
+	for valid {
+
+		// stop when
+		if !bytes.HasPrefix(key, keyPrefix) {
+			break
+		}
+		var fieldRow *FieldRow
+		fieldRow, err = NewFieldRowKV(key, val)
+		if err != nil {
+			return
+		}
+		udc.fieldIndexCache.AddExisting(fieldRow.name, fieldRow.index)
+
+		it.Next()
+		key, val, valid = it.Current()
+	}
+
+	keyPrefix = []byte{'v'}
+	val, err = kvreader.Get(keyPrefix)
+	if err != nil {
+		return
+	}
+	var vr *VersionRow
+	vr, err = NewVersionRowKV(keyPrefix, val)
+	if err != nil {
+		return
+	}
+	if vr.version != Version {
+		err = IncompatibleVersion
+		return
+	}
+
+	return
+}
+
+func (udc *UpsideDownCouch) batchRows(writer store.KVWriter, addRows []UpsideDownCouchRow, updateRows []UpsideDownCouchRow, deleteRows []UpsideDownCouchRow) (err error) {
+
+	// prepare batch
+	wb := writer.NewBatch()
+
+	// add
+	for _, row := range addRows {
+		tfr, ok := row.(*TermFrequencyRow)
+		if ok {
+			// need to increment counter
+			dictionaryKey := tfr.DictionaryRowKey()
+			wb.Merge(dictionaryKey, dictionaryTermIncr)
+		}
+		wb.Set(row.Key(), row.Value())
+	}
+
+	// update
+	for _, row := range updateRows {
+		wb.Set(row.Key(), row.Value())
+	}
+
+	// delete
+	for _, row := range deleteRows {
+		tfr, ok := row.(*TermFrequencyRow)
+		if ok {
+			// need to decrement counter
+			dictionaryKey := tfr.DictionaryRowKey()
+			wb.Merge(dictionaryKey, dictionaryTermDecr)
+		}
+		wb.Delete(row.Key())
+	}
+
+	// write out the batch
+	err = wb.Execute()
+	if err != nil {
+		return
+	}
+	return
+}
+
+func (udc *UpsideDownCouch) DocCount() (uint64, error) {
+	return udc.docCount, nil
+}
+
+func (udc *UpsideDownCouch) Open() (err error) {
+	// install the merge operator
+	udc.store.SetMergeOperator(&mergeOperator)
+
+	// now open the kv store
+	err = udc.store.Open()
+	if err != nil {
+		return
+	}
+
+	// start a writer for the open process
+	var kvwriter store.KVWriter
+	kvwriter, err = udc.store.Writer()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if cerr := kvwriter.Close(); err == nil && cerr != nil {
+			err = cerr
+		}
+	}()
+
+	var value []byte
+	value, err = kvwriter.Get(VersionKey)
+	if err != nil {
+		return
+	}
+
+	// init new index OR load schema
+	if value == nil {
+		err = udc.init(kvwriter)
+		if err != nil {
+			return
+		}
+	} else {
+		err = udc.loadSchema(kvwriter)
+		if err != nil {
+			return
+		}
+	}
+	// set doc count
+	udc.docCount, err = udc.countDocs(kvwriter)
+	return
+}
+
+func (udc *UpsideDownCouch) countDocs(kvreader store.KVReader) (count uint64, err error) {
+	it := kvreader.Iterator([]byte{'b'})
+	defer func() {
+		if cerr := it.Close(); err == nil && cerr != nil {
+			err = cerr
+		}
+	}()
+
+	key, _, valid := it.Current()
+	for valid {
+		if !bytes.HasPrefix(key, []byte{'b'}) {
+			break
+		}
+		count++
+		it.Next()
+		key, _, valid = it.Current()
+	}
+
+	return
+}
+
+func (udc *UpsideDownCouch) rowCount() (count uint64, err error) {
+	// start an isolated reader for use during the rowcount
+	kvreader, err := udc.store.Reader()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if cerr := kvreader.Close(); err == nil && cerr != nil {
+			err = cerr
+		}
+	}()
+	it := kvreader.Iterator([]byte{0})
+	defer func() {
+		if cerr := it.Close(); err == nil && cerr != nil {
+			err = cerr
+		}
+	}()
+
+	_, _, valid := it.Current()
+	for valid {
+		count++
+		it.Next()
+		_, _, valid = it.Current()
+	}
+
+	return
+}
+
+func (udc *UpsideDownCouch) Close() error {
+	return udc.store.Close()
+}
+
+func (udc *UpsideDownCouch) Update(doc *document.Document) (err error) {
+	// do analysis before acquiring write lock
+	analysisStart := time.Now()
+	resultChan := make(chan *AnalysisResult)
+	aw := AnalysisWork{
+		udc: udc,
+		d:   doc,
+		rc:  resultChan,
+	}
+	// put the work on the queue
+	go func() {
+		udc.analysisQueue.Queue(&aw)
+	}()
+
+	// wait for the result
+	result := <-resultChan
+	close(resultChan)
+	atomic.AddUint64(&udc.stats.analysisTime, uint64(time.Since(analysisStart)))
+
+	// start a writer for this update
+	indexStart := time.Now()
+	var kvwriter store.KVWriter
+	kvwriter, err = udc.store.Writer()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if cerr := kvwriter.Close(); err == nil && cerr != nil {
+			err = cerr
+		}
+	}()
+
+	// first we lookup the backindex row for the doc id if it exists
+	// lookup the back index row
+	var backIndexRow *BackIndexRow
+	backIndexRow, err = udc.backIndexRowForDoc(kvwriter, doc.ID)
+	if err != nil {
+		atomic.AddUint64(&udc.stats.errors, 1)
+		return
+	}
+
+	// prepare a list of rows
+	addRows := make([]UpsideDownCouchRow, 0)
+	updateRows := make([]UpsideDownCouchRow, 0)
+	deleteRows := make([]UpsideDownCouchRow, 0)
+
+	addRows, updateRows, deleteRows = udc.mergeOldAndNew(backIndexRow, result.rows, addRows, updateRows, deleteRows)
+
+	err = udc.batchRows(kvwriter, addRows, updateRows, deleteRows)
+	if err == nil && backIndexRow == nil {
+		udc.docCount++
+	}
+	atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart)))
+	if err == nil {
+		atomic.AddUint64(&udc.stats.updates, 1)
+	} else {
+		atomic.AddUint64(&udc.stats.errors, 1)
+	}
+	return
+}
+
+func (udc *UpsideDownCouch) mergeOldAndNew(backIndexRow *BackIndexRow, rows, addRows, updateRows, deleteRows []UpsideDownCouchRow) ([]UpsideDownCouchRow, []UpsideDownCouchRow, []UpsideDownCouchRow) {
+	existingTermKeys := make(map[string]bool)
+	for _, key := range backIndexRow.AllTermKeys() {
+		existingTermKeys[string(key)] = true
+	}
+
+	existingStoredKeys := make(map[string]bool)
+	for _, key := range backIndexRow.AllStoredKeys() {
+		existingStoredKeys[string(key)] = true
+	}
+
+	for _, row := range rows {
+		switch row := row.(type) {
+		case *TermFrequencyRow:
+			rowKey := string(row.Key())
+			if _, ok := existingTermKeys[rowKey]; ok {
+				updateRows = append(updateRows, row)
+				delete(existingTermKeys, rowKey)
+			} else {
+				addRows = append(addRows, row)
+			}
+		case *StoredRow:
+			rowKey := string(row.Key())
+			if _, ok := existingStoredKeys[rowKey]; ok {
+				updateRows = append(updateRows, row)
+				delete(existingStoredKeys, rowKey)
+			} else {
+				addRows = append(addRows, row)
+			}
+		default:
+			updateRows = append(updateRows, row)
+		}
+
+	}
+
+	// any of the existing rows that weren't updated need to be deleted
+	for existingTermKey := range existingTermKeys {
+		termFreqRow, err := NewTermFrequencyRowK([]byte(existingTermKey))
+		if err == nil {
+			deleteRows = append(deleteRows, termFreqRow)
+		}
+	}
+
+	// any of the existing stored fields that weren't updated need to be deleted
+	for existingStoredKey := range existingStoredKeys {
+		storedRow, err := NewStoredRowK([]byte(existingStoredKey))
+		if err == nil {
+			deleteRows = append(deleteRows, storedRow)
+		}
+	}
+
+	return addRows, updateRows, deleteRows
+}
+
+func (udc *UpsideDownCouch) storeField(docID string, field document.Field, fieldIndex uint16) ([]UpsideDownCouchRow, []*BackIndexStoreEntry) {
+	rows := make([]UpsideDownCouchRow, 0, 100)
+	backIndexStoredEntries := make([]*BackIndexStoreEntry, 0)
+	fieldType := encodeFieldType(field)
+	storedRow := NewStoredRow(docID, fieldIndex, field.ArrayPositions(), fieldType, field.Value())
+
+	// record the back index entry
+	backIndexStoredEntry := BackIndexStoreEntry{Field: proto.Uint32(uint32(fieldIndex)), ArrayPositions: field.ArrayPositions()}
+	backIndexStoredEntries = append(backIndexStoredEntries, &backIndexStoredEntry)
+
+	rows = append(rows, storedRow)
+	return rows, backIndexStoredEntries
+}
+
+func encodeFieldType(f document.Field) byte {
+	fieldType := byte('x')
+	switch f.(type) {
+	case *document.TextField:
+		fieldType = 't'
+	case *document.NumericField:
+		fieldType = 'n'
+	case *document.DateTimeField:
+		fieldType = 'd'
+	case *document.CompositeField:
+		fieldType = 'c'
+	}
+	return fieldType
+}
+
+func (udc *UpsideDownCouch) indexField(docID string, field document.Field, fieldIndex uint16, fieldLength int, tokenFreqs analysis.TokenFrequencies) ([]UpsideDownCouchRow, []*BackIndexTermEntry) {
+
+	rows := make([]UpsideDownCouchRow, 0, 100)
+	backIndexTermEntries := make([]*BackIndexTermEntry, 0)
+	fieldNorm := float32(1.0 / math.Sqrt(float64(fieldLength)))
+
+	for _, tf := range tokenFreqs {
+		var termFreqRow *TermFrequencyRow
+		if field.Options().IncludeTermVectors() {
+			tv, newFieldRows := udc.termVectorsFromTokenFreq(fieldIndex, tf)
+			rows = append(rows, newFieldRows...)
+			termFreqRow = NewTermFrequencyRowWithTermVectors(tf.Term, fieldIndex, docID, uint64(frequencyFromTokenFreq(tf)), fieldNorm, tv)
+		} else {
+			termFreqRow = NewTermFrequencyRow(tf.Term, fieldIndex, docID, uint64(frequencyFromTokenFreq(tf)), fieldNorm)
+		}
+
+		// record the back index entry
+		backIndexTermEntry := BackIndexTermEntry{Term: proto.String(string(tf.Term)), Field: proto.Uint32(uint32(fieldIndex))}
+		backIndexTermEntries = append(backIndexTermEntries, &backIndexTermEntry)
+
+		rows = append(rows, termFreqRow)
+	}
+
+	return rows, backIndexTermEntries
+}
+
+func (udc *UpsideDownCouch) Delete(id string) (err error) {
+	indexStart := time.Now()
+	// start a writer for this delete
+	var kvwriter store.KVWriter
+	kvwriter, err = udc.store.Writer()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if cerr := kvwriter.Close(); err == nil && cerr != nil {
+			err = cerr
+		}
+	}()
+
+	// lookup the back index row
+	var backIndexRow *BackIndexRow
+	backIndexRow, err = udc.backIndexRowForDoc(kvwriter, id)
+	if err != nil {
+		atomic.AddUint64(&udc.stats.errors, 1)
+		return
+	}
+	if backIndexRow == nil {
+		atomic.AddUint64(&udc.stats.deletes, 1)
+		return
+	}
+
+	deleteRows := make([]UpsideDownCouchRow, 0)
+	deleteRows = udc.deleteSingle(id, backIndexRow, deleteRows)
+
+	err = udc.batchRows(kvwriter, nil, nil, deleteRows)
+	if err == nil {
+		udc.docCount--
+	}
+	atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart)))
+	if err == nil {
+		atomic.AddUint64(&udc.stats.deletes, 1)
+	} else {
+		atomic.AddUint64(&udc.stats.errors, 1)
+	}
+	return
+}
+
+func (udc *UpsideDownCouch) deleteSingle(id string, backIndexRow *BackIndexRow, deleteRows []UpsideDownCouchRow) []UpsideDownCouchRow {
+
+	for _, backIndexEntry := range backIndexRow.termEntries {
+		tfr := NewTermFrequencyRow([]byte(*backIndexEntry.Term), uint16(*backIndexEntry.Field), id, 0, 0)
+		deleteRows = append(deleteRows, tfr)
+	}
+	for _, se := range backIndexRow.storedEntries {
+		sf := NewStoredRow(id, uint16(*se.Field), se.ArrayPositions, 'x', nil)
+		deleteRows = append(deleteRows, sf)
+	}
+
+	// also delete the back entry itself
+	deleteRows = append(deleteRows, backIndexRow)
+	return deleteRows
+}
+
+func (udc *UpsideDownCouch) backIndexRowForDoc(kvreader store.KVReader, docID string) (*BackIndexRow, error) {
+	// use a temporary row structure to build key
+	tempRow := &BackIndexRow{
+		doc: []byte(docID),
+	}
+	key := tempRow.Key()
+	value, err := kvreader.Get(key)
+	if err != nil {
+		return nil, err
+	}
+	if value == nil {
+		return nil, nil
+	}
+	backIndexRow, err := NewBackIndexRowKV(key, value)
+	if err != nil {
+		return nil, err
+	}
+	return backIndexRow, nil
+}
+
+func (udc *UpsideDownCouch) backIndexRowsForBatch(kvreader store.KVReader, batch *index.Batch) (map[string]*BackIndexRow, error) {
+	// FIXME faster to order the ids and scan sequentially
+	// for now just get it working
+	rv := make(map[string]*BackIndexRow, 0)
+	for docID := range batch.IndexOps {
+		backIndexRow, err := udc.backIndexRowForDoc(kvreader, docID)
+		if err != nil {
+			return nil, err
+		}
+		rv[docID] = backIndexRow
+	}
+	return rv, nil
+}
+
+func decodeFieldType(typ byte, name string, value []byte) document.Field {
+	switch typ {
+	case 't':
+		return document.NewTextField(name, []uint64{}, value)
+	case 'n':
+		return document.NewNumericFieldFromBytes(name, []uint64{}, value)
+	case 'd':
+		return document.NewDateTimeFieldFromBytes(name, []uint64{}, value)
+	}
+	return nil
+}
+
+func frequencyFromTokenFreq(tf *analysis.TokenFreq) int {
+	return len(tf.Locations)
+}
+
+func (udc *UpsideDownCouch) termVectorsFromTokenFreq(field uint16, tf *analysis.TokenFreq) ([]*TermVector, []UpsideDownCouchRow) {
+	rv := make([]*TermVector, len(tf.Locations))
+	newFieldRows := make([]UpsideDownCouchRow, 0)
+
+	for i, l := range tf.Locations {
+		var newFieldRow *FieldRow
+		fieldIndex := field
+		if l.Field != "" {
+			// lookup correct field
+			fieldIndex, newFieldRow = udc.fieldIndexCache.FieldIndex(l.Field)
+			if newFieldRow != nil {
+				newFieldRows = append(newFieldRows, newFieldRow)
+			}
+		}
+		tv := TermVector{
+			field: fieldIndex,
+			pos:   uint64(l.Position),
+			start: uint64(l.Start),
+			end:   uint64(l.End),
+		}
+		rv[i] = &tv
+	}
+
+	return rv, newFieldRows
+}
+
+func (udc *UpsideDownCouch) termFieldVectorsFromTermVectors(in []*TermVector) []*index.TermFieldVector {
+	rv := make([]*index.TermFieldVector, len(in))
+
+	for i, tv := range in {
+		fieldName := udc.fieldIndexCache.FieldName(tv.field)
+		tfv := index.TermFieldVector{
+			Field: fieldName,
+			Pos:   tv.pos,
+			Start: tv.start,
+			End:   tv.end,
+		}
+		rv[i] = &tfv
+	}
+	return rv
+}
+
+func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) {
+	analysisStart := time.Now()
+	resultChan := make(chan *AnalysisResult)
+
+	var numUpdates uint64
+	for _, doc := range batch.IndexOps {
+		if doc != nil {
+			numUpdates++
+		}
+	}
+
+	go func() {
+		for _, doc := range batch.IndexOps {
+			if doc != nil {
+				aw := AnalysisWork{
+					udc: udc,
+					d:   doc,
+					rc:  resultChan,
+				}
+				// put the work on the queue
+				udc.analysisQueue.Queue(&aw)
+			}
+		}
+	}()
+
+	newRowsMap := make(map[string][]UpsideDownCouchRow)
+	// wait for the result
+	var itemsDeQueued uint64
+	for itemsDeQueued < numUpdates {
+		result := <-resultChan
+		newRowsMap[result.docID] = result.rows
+		itemsDeQueued++
+	}
+	close(resultChan)
+
+	atomic.AddUint64(&udc.stats.analysisTime, uint64(time.Since(analysisStart)))
+
+	indexStart := time.Now()
+	// start a writer for this batch
+	var kvwriter store.KVWriter
+	kvwriter, err = udc.store.Writer()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if cerr := kvwriter.Close(); err == nil && cerr != nil {
+			err = cerr
+		}
+	}()
+
+	// first lookup all the back index rows
+	var backIndexRows map[string]*BackIndexRow
+	backIndexRows, err = udc.backIndexRowsForBatch(kvwriter, batch)
+	if err != nil {
+		return
+	}
+
+	// prepare a list of rows
+	addRows := make([]UpsideDownCouchRow, 0)
+	updateRows := make([]UpsideDownCouchRow, 0)
+	deleteRows := make([]UpsideDownCouchRow, 0)
+
+	docsAdded := uint64(0)
+	docsDeleted := uint64(0)
+	for docID, doc := range batch.IndexOps {
+		backIndexRow := backIndexRows[docID]
+		if doc == nil && backIndexRow != nil {
+			// delete
+			deleteRows = udc.deleteSingle(docID, backIndexRow, deleteRows)
+			docsDeleted++
+		} else if doc != nil {
+			addRows, updateRows, deleteRows = udc.mergeOldAndNew(backIndexRow, newRowsMap[docID], addRows, updateRows, deleteRows)
+			if backIndexRow == nil {
+				docsAdded++
+			}
+		}
+	}
+
+	// add the internal ops
+	for internalKey, internalValue := range batch.InternalOps {
+		if internalValue == nil {
+			// delete
+			deleteInternalRow := NewInternalRow([]byte(internalKey), nil)
+			deleteRows = append(deleteRows, deleteInternalRow)
+		} else {
+			updateInternalRow := NewInternalRow([]byte(internalKey), internalValue)
+			updateRows = append(updateRows, updateInternalRow)
+		}
+	}
+
+	err = udc.batchRows(kvwriter, addRows, updateRows, deleteRows)
+	atomic.AddUint64(&udc.stats.indexTime, uint64(time.Since(indexStart)))
+	if err == nil {
+		udc.docCount += docsAdded
+		udc.docCount -= docsDeleted
+		atomic.AddUint64(&udc.stats.updates, numUpdates)
+		atomic.AddUint64(&udc.stats.deletes, docsDeleted)
+		atomic.AddUint64(&udc.stats.batches, 1)
+	} else {
+		atomic.AddUint64(&udc.stats.errors, 1)
+	}
+	return
+}
+
+func (udc *UpsideDownCouch) SetInternal(key, val []byte) (err error) {
+	internalRow := NewInternalRow(key, val)
+	var writer store.KVWriter
+	writer, err = udc.store.Writer()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if cerr := writer.Close(); err == nil && cerr != nil {
+			err = cerr
+		}
+	}()
+	return writer.Set(internalRow.Key(), internalRow.Value())
+}
+
+func (udc *UpsideDownCouch) DeleteInternal(key []byte) (err error) {
+	internalRow := NewInternalRow(key, nil)
+	var writer store.KVWriter
+	writer, err = udc.store.Writer()
+	if err != nil {
+		return
+	}
+	defer func() {
+		if cerr := writer.Close(); err == nil && cerr != nil {
+			err = cerr
+		}
+	}()
+	return writer.Delete(internalRow.Key())
+}
+
+func (udc *UpsideDownCouch) Reader() (index.IndexReader, error) {
+	kvr, err := udc.store.Reader()
+	if err != nil {
+		return nil, fmt.Errorf("error opening store reader: %v", err)
+	}
+	return &IndexReader{
+		index:    udc,
+		kvreader: kvr,
+		docCount: udc.docCount,
+	}, nil
+}
+
+func (udc *UpsideDownCouch) Stats() json.Marshaler {
+	return udc.stats
+}

vendor/github.com/blevesearch/bleve/index/upside_down/upside_down.pb.go

@@ -0,0 +1,98 @@
+// Code generated by protoc-gen-go.
+// source: upside_down.proto
+// DO NOT EDIT!
+
+/*
+Package upside_down is a generated protocol buffer package.
+
+It is generated from these files:
+	upside_down.proto
+
+It has these top-level messages:
+	BackIndexTermEntry
+	BackIndexStoreEntry
+	BackIndexRowValue
+*/
+package upside_down
+
+import proto "github.com/golang/protobuf/proto"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = math.Inf
+
+type BackIndexTermEntry struct {
+	Term             *string `protobuf:"bytes,1,req,name=term" json:"term,omitempty"`
+	Field            *uint32 `protobuf:"varint,2,req,name=field" json:"field,omitempty"`
+	XXX_unrecognized []byte  `json:"-"`
+}
+
+func (m *BackIndexTermEntry) Reset()         { *m = BackIndexTermEntry{} }
+func (m *BackIndexTermEntry) String() string { return proto.CompactTextString(m) }
+func (*BackIndexTermEntry) ProtoMessage()    {}
+
+func (m *BackIndexTermEntry) GetTerm() string {
+	if m != nil && m.Term != nil {
+		return *m.Term
+	}
+	return ""
+}
+
+func (m *BackIndexTermEntry) GetField() uint32 {
+	if m != nil && m.Field != nil {
+		return *m.Field
+	}
+	return 0
+}
+
+type BackIndexStoreEntry struct {
+	Field            *uint32  `protobuf:"varint,1,req,name=field" json:"field,omitempty"`
+	ArrayPositions   []uint64 `protobuf:"varint,2,rep,name=arrayPositions" json:"arrayPositions,omitempty"`
+	XXX_unrecognized []byte   `json:"-"`
+}
+
+func (m *BackIndexStoreEntry) Reset()         { *m = BackIndexStoreEntry{} }
+func (m *BackIndexStoreEntry) String() string { return proto.CompactTextString(m) }
+func (*BackIndexStoreEntry) ProtoMessage()    {}
+
+func (m *BackIndexStoreEntry) GetField() uint32 {
+	if m != nil && m.Field != nil {
+		return *m.Field
+	}
+	return 0
+}
+
+func (m *BackIndexStoreEntry) GetArrayPositions() []uint64 {
+	if m != nil {
+		return m.ArrayPositions
+	}
+	return nil
+}
+
+type BackIndexRowValue struct {
+	TermEntries      []*BackIndexTermEntry  `protobuf:"bytes,1,rep,name=termEntries" json:"termEntries,omitempty"`
+	StoredEntries    []*BackIndexStoreEntry `protobuf:"bytes,2,rep,name=storedEntries" json:"storedEntries,omitempty"`
+	XXX_unrecognized []byte                 `json:"-"`
+}
+
+func (m *BackIndexRowValue) Reset()         { *m = BackIndexRowValue{} }
+func (m *BackIndexRowValue) String() string { return proto.CompactTextString(m) }
+func (*BackIndexRowValue) ProtoMessage()    {}
+
+func (m *BackIndexRowValue) GetTermEntries() []*BackIndexTermEntry {
+	if m != nil {
+		return m.TermEntries
+	}
+	return nil
+}
+
+func (m *BackIndexRowValue) GetStoredEntries() []*BackIndexStoreEntry {
+	if m != nil {
+		return m.StoredEntries
+	}
+	return nil
+}
+
+func init() {
+}

vendor/github.com/blevesearch/bleve/index/upside_down/upside_down.proto

@@ -0,0 +1,14 @@
+message BackIndexTermEntry {
+	required string term = 1;
+	required uint32 field = 2;
+}
+
+message BackIndexStoreEntry {
+	required uint32 field = 1;
+	repeated uint64 arrayPositions = 2;
+}
+
+message BackIndexRowValue {
+	repeated BackIndexTermEntry termEntries = 1;
+	repeated BackIndexStoreEntry storedEntries = 2;
+}