e2m/dispatch
4
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Upgrade server dependencies, manage them with govendor

Browse source
This commit is contained in:
Ken-Håvard Lieng 2017-04-18 03:02:51 +02:00
parent ebee2746d6
commit 971278e7e5
1748 changed files with 196165 additions and 194500 deletions
Download patch file Download diff file Expand all files Collapse all files

71
vendor/github.com/blevesearch/bleve/index/store/batch.go generated vendored
View file

@ -1,11 +1,16 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package store
@ -15,53 +20,41 @@ type op struct {
}
type EmulatedBatch struct {
w KVWriter
ops []*op
merge *EmulatedMerge
Ops []*op
Merger *EmulatedMerge
}
func NewEmulatedBatch(w KVWriter, mo MergeOperator) *EmulatedBatch {
func NewEmulatedBatch(mo MergeOperator) *EmulatedBatch {
return &EmulatedBatch{
w: w,
ops: make([]*op, 0, 1000),
merge: NewEmulatedMerge(mo),
Ops: make([]*op, 0, 1000),
Merger: NewEmulatedMerge(mo),
}
}
func (b *EmulatedBatch) Set(key, val []byte) {
b.ops = append(b.ops, &op{key, val})
ck := make([]byte, len(key))
copy(ck, key)
cv := make([]byte, len(val))
copy(cv, val)
b.Ops = append(b.Ops, &op{ck, cv})
}
func (b *EmulatedBatch) Delete(key []byte) {
b.ops = append(b.ops, &op{key, nil})
ck := make([]byte, len(key))
copy(ck, key)
b.Ops = append(b.Ops, &op{ck, nil})
}
func (b *EmulatedBatch) Merge(key, val []byte) {
b.merge.Merge(key, val)
ck := make([]byte, len(key))
copy(ck, key)
cv := make([]byte, len(val))
copy(cv, val)
b.Merger.Merge(key, val)
}
func (b *EmulatedBatch) Execute() error {
// first process merges
err := b.merge.Execute(b.w)
if err != nil {
return err
}
// now apply all the ops
for _, op := range b.ops {
if op.V != nil {
err := b.w.Set(op.K, op.V)
if err != nil {
return err
}
} else {
err := b.w.Delete(op.K)
if err != nil {
return err
}
}
}
return nil
func (b *EmulatedBatch) Reset() {
b.Ops = b.Ops[:0]
}
func (b *EmulatedBatch) Close() error {

49
vendor/github.com/blevesearch/bleve/index/store/boltdb/iterator.go generated vendored
View file

@ -1,15 +1,22 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package boltdb
import (
"bytes"
"github.com/boltdb/bolt"
)
@ -17,24 +24,44 @@ type Iterator struct {
store *Store
tx *bolt.Tx
cursor *bolt.Cursor
prefix []byte
start []byte
end []byte
valid bool
key []byte
val []byte
}
func (i *Iterator) SeekFirst() {
i.key, i.val = i.cursor.First()
func (i *Iterator) updateValid() {
i.valid = (i.key != nil)
if i.valid {
if i.prefix != nil {
i.valid = bytes.HasPrefix(i.key, i.prefix)
} else if i.end != nil {
i.valid = bytes.Compare(i.key, i.end) < 0
}
}
}
func (i *Iterator) Seek(k []byte) {
if i.start != nil && bytes.Compare(k, i.start) < 0 {
k = i.start
}
if i.prefix != nil && !bytes.HasPrefix(k, i.prefix) {
if bytes.Compare(k, i.prefix) < 0 {
k = i.prefix
} else {
i.valid = false
return
}
}
i.key, i.val = i.cursor.Seek(k)
i.valid = (i.key != nil)
i.updateValid()
}
func (i *Iterator) Next() {
i.key, i.val = i.cursor.Next()
i.valid = (i.key != nil)
i.updateValid()
}
func (i *Iterator) Current() ([]byte, []byte, bool) {

62
vendor/github.com/blevesearch/bleve/index/store/boltdb/reader.go generated vendored
View file

@ -1,11 +1,16 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package boltdb
@ -15,30 +20,51 @@ import (
)
type Reader struct {
store *Store
tx *bolt.Tx
}
func (r *Reader) BytesSafeAfterClose() bool {
return false
store *Store
tx *bolt.Tx
bucket *bolt.Bucket
}
func (r *Reader) Get(key []byte) ([]byte, error) {
rv := r.tx.Bucket([]byte(r.store.bucket)).Get(key)
var rv []byte
v := r.bucket.Get(key)
if v != nil {
rv = make([]byte, len(v))
copy(rv, v)
}
return rv, nil
}
func (r *Reader) Iterator(key []byte) store.KVIterator {
b := r.tx.Bucket([]byte(r.store.bucket))
cursor := b.Cursor()
func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {
return store.MultiGet(r, keys)
}
func (r *Reader) PrefixIterator(prefix []byte) store.KVIterator {
cursor := r.bucket.Cursor()
rv := &Iterator{
store: r.store,
tx: r.tx,
cursor: cursor,
prefix: prefix,
}
rv.Seek(key)
rv.Seek(prefix)
return rv
}
func (r *Reader) RangeIterator(start, end []byte) store.KVIterator {
cursor := r.bucket.Cursor()
rv := &Iterator{
store: r.store,
tx: r.tx,
cursor: cursor,
start: start,
end: end,
}
rv.Seek(start)
return rv
}

26
vendor/github.com/blevesearch/bleve/index/store/boltdb/stats.go generated vendored Normal file
View file

@ -0,0 +1,26 @@
// Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package boltdb
import "encoding/json"
type stats struct {
s *Store
}
func (s *stats) MarshalJSON() ([]byte, error) {
bs := s.s.db.Stats()
return json.Marshal(bs)
}

195
vendor/github.com/blevesearch/bleve/index/store/boltdb/store.go generated vendored
View file

@ -1,63 +1,105 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package boltdb implements a store.KVStore on top of BoltDB. It supports the
// following options:
//
// "bucket" (string): the name of BoltDB bucket to use, defaults to "bleve".
//
// "nosync" (bool): if true, set boltdb.DB.NoSync to true. It speeds up index
// operations in exchange of losing integrity guarantees if indexation aborts
// without closing the index. Use it when rebuilding indexes from zero.
package boltdb
import (
"bytes"
"encoding/json"
"fmt"
"sync"
"os"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/boltdb/bolt"
)
const Name = "boltdb"
const (
Name = "boltdb"
defaultCompactBatchSize = 100
)
type Store struct {
path string
bucket string
db *bolt.DB
writer sync.Mutex
mo store.MergeOperator
path string
bucket string
db *bolt.DB
noSync bool
fillPercent float64
mo store.MergeOperator
}
func New(path string, bucket string) *Store {
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
}
if path == "" {
return nil, os.ErrInvalid
}
bucket, ok := config["bucket"].(string)
if !ok {
bucket = "bleve"
}
noSync, _ := config["nosync"].(bool)
fillPercent, ok := config["fillPercent"].(float64)
if !ok {
fillPercent = bolt.DefaultFillPercent
}
bo := &bolt.Options{}
ro, ok := config["read_only"].(bool)
if ok {
bo.ReadOnly = ro
}
db, err := bolt.Open(path, 0600, bo)
if err != nil {
return nil, err
}
db.NoSync = noSync
if !bo.ReadOnly {
err = db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(bucket))
return err
})
if err != nil {
return nil, err
}
}
rv := Store{
path: path,
bucket: bucket,
path: path,
bucket: bucket,
db: db,
mo: mo,
noSync: noSync,
fillPercent: fillPercent,
}
return &rv
}
func (bs *Store) Open() error {
var err error
bs.db, err = bolt.Open(bs.path, 0600, nil)
if err != nil {
return err
}
err = bs.db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(bs.bucket))
return err
})
if err != nil {
return err
}
return nil
}
func (bs *Store) SetMergeOperator(mo store.MergeOperator) {
bs.mo = mo
return &rv, nil
}
func (bs *Store) Close() error {
@ -70,43 +112,64 @@ func (bs *Store) Reader() (store.KVReader, error) {
return nil, err
}
return &Reader{
store: bs,
tx: tx,
store: bs,
tx: tx,
bucket: tx.Bucket([]byte(bs.bucket)),
}, nil
}
func (bs *Store) Writer() (store.KVWriter, error) {
bs.writer.Lock()
tx, err := bs.db.Begin(true)
if err != nil {
bs.writer.Unlock()
return nil, err
}
reader := &Reader{
store: bs,
tx: tx,
}
return &Writer{
store: bs,
tx: tx,
reader: reader,
store: bs,
}, nil
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
func (bs *Store) Stats() json.Marshaler {
return &stats{
s: bs,
}
}
bucket, ok := config["bucket"].(string)
if !ok {
bucket = "bleve"
// CompactWithBatchSize removes DictionaryTerm entries with a count of zero (in batchSize batches)
// Removing entries is a workaround for github issue #374.
func (bs *Store) CompactWithBatchSize(batchSize int) error {
for {
cnt := 0
err := bs.db.Batch(func(tx *bolt.Tx) error {
c := tx.Bucket([]byte(bs.bucket)).Cursor()
prefix := []byte("d")
for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
if bytes.Equal(v, []byte{0}) {
cnt++
if err := c.Delete(); err != nil {
return err
}
if cnt == batchSize {
break
}
}
}
return nil
})
if err != nil {
return err
}
if cnt == 0 {
break
}
}
return nil
}
return New(path, bucket), nil
// Compact calls CompactWithBatchSize with a default batch size of 100. This is a workaround
// for github issue #374.
func (bs *Store) Compact() error {
return bs.CompactWithBatchSize(defaultCompactBatchSize)
}
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
registry.RegisterKVStore(Name, New)
}

272
vendor/github.com/blevesearch/bleve/index/store/boltdb/store_test.go generated vendored
View file

@ -1,272 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package boltdb
import (
"os"
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
)
func TestStore(t *testing.T) {
s := New("test", "bleve")
err := s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
CommonTestKVStore(t, s)
}
func TestReaderIsolation(t *testing.T) {
s := New("test", "bleve")
err := s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
CommonTestReaderIsolation(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
}
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
// insert a kv pair
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// create an isolated reader
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// verify that we see the value already inserted
val, err := reader.Get([]byte("a"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-a")) {
t.Errorf("expected val-a, got nil")
}
// verify that an iterator sees it
count := 0
it := reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
// add something after the reader was created
writer, err = s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("b"), []byte("val-b"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// ensure that a newer reader sees it
newReader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := newReader.Close()
if err != nil {
t.Fatal(err)
}
}()
val, err = newReader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-b")) {
t.Errorf("expected val-b, got nil")
}
// ensure that the director iterator sees it
count = 0
it2 := newReader.Iterator([]byte{0})
defer func() {
err := it2.Close()
if err != nil {
t.Fatal(err)
}
}()
for it2.Valid() {
it2.Next()
count++
}
if count != 2 {
t.Errorf("expected iterator to see 2, saw %d", count)
}
// but that the isolated reader does not
val, err = reader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %v", val)
}
// and ensure that the iterator on the isolated reader also does not
count = 0
it3 := reader.Iterator([]byte{0})
defer func() {
err := it3.Close()
if err != nil {
t.Fatal(err)
}
}()
for it3.Valid() {
it3.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
}

113
vendor/github.com/blevesearch/bleve/index/store/boltdb/writer.go generated vendored
View file

@ -1,50 +1,95 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package boltdb
import (
"fmt"
"github.com/blevesearch/bleve/index/store"
"github.com/boltdb/bolt"
)
type Writer struct {
store *Store
tx *bolt.Tx
reader *Reader
}
func (w *Writer) Set(key, val []byte) error {
return w.tx.Bucket([]byte(w.store.bucket)).Put(key, val)
}
func (w *Writer) Delete(key []byte) error {
return w.tx.Bucket([]byte(w.store.bucket)).Delete(key)
store *Store
}
func (w *Writer) NewBatch() store.KVBatch {
return store.NewEmulatedBatch(w, w.store.mo)
return store.NewEmulatedBatch(w.store.mo)
}
func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) {
return make([]byte, options.TotalBytes), w.NewBatch(), nil
}
func (w *Writer) ExecuteBatch(batch store.KVBatch) (err error) {
emulatedBatch, ok := batch.(*store.EmulatedBatch)
if !ok {
return fmt.Errorf("wrong type of batch")
}
tx, err := w.store.db.Begin(true)
if err != nil {
return
}
// defer function to ensure that once started,
// we either Commit tx or Rollback
defer func() {
// if nothing went wrong, commit
if err == nil {
// careful to catch error here too
err = tx.Commit()
} else {
// caller should see error that caused abort,
// not success or failure of Rollback itself
_ = tx.Rollback()
}
}()
bucket := tx.Bucket([]byte(w.store.bucket))
bucket.FillPercent = w.store.fillPercent
for k, mergeOps := range emulatedBatch.Merger.Merges {
kb := []byte(k)
existingVal := bucket.Get(kb)
mergedVal, fullMergeOk := w.store.mo.FullMerge(kb, existingVal, mergeOps)
if !fullMergeOk {
err = fmt.Errorf("merge operator returned failure")
return
}
err = bucket.Put(kb, mergedVal)
if err != nil {
return
}
}
for _, op := range emulatedBatch.Ops {
if op.V != nil {
err = bucket.Put(op.K, op.V)
if err != nil {
return
}
} else {
err = bucket.Delete(op.K)
if err != nil {
return
}
}
}
return
}
func (w *Writer) Close() error {
w.store.writer.Unlock()
return w.tx.Commit()
}
func (w *Writer) BytesSafeAfterClose() bool {
return w.reader.BytesSafeAfterClose()
}
func (w *Writer) Get(key []byte) ([]byte, error) {
return w.reader.Get(key)
}
func (w *Writer) Iterator(key []byte) store.KVIterator {
return w.reader.Iterator(key)
return nil
}

88
vendor/github.com/blevesearch/bleve/index/store/cznicb/batch.go generated vendored
View file

@ -1,88 +0,0 @@
// Copyright (c) 2015 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
// +build go1.4
package cznicb
import ()
type op struct {
k []byte
v []byte
}
type Batch struct {
s *Store
ops []op
merges map[string][][]byte
}
func (b *Batch) Set(k, v []byte) {
b.ops = append(b.ops, op{k, v})
}
func (b *Batch) Delete(k []byte) {
b.ops = append(b.ops, op{k, nil})
}
func (b *Batch) Merge(key, val []byte) {
ops, ok := b.merges[string(key)]
if ok && len(ops) > 0 {
last := ops[len(ops)-1]
mergedVal, partialMergeOk := b.s.mo.PartialMerge(key, last, val)
if partialMergeOk {
// replace last entry with the result of the merge
ops[len(ops)-1] = mergedVal
} else {
// could not partial merge, append this to the end
ops = append(ops, val)
}
} else {
ops = [][]byte{val}
}
b.merges[string(key)] = ops
}
func (b *Batch) Execute() (err error) {
b.s.m.Lock()
defer b.s.m.Unlock()
t := b.s.t
for key, mergeOps := range b.merges {
k := []byte(key)
t.Put(k, func(oldV interface{}, exists bool) (newV interface{}, write bool) {
ob := []byte(nil)
if exists && oldV != nil {
ob = oldV.([]byte)
}
mergedVal, fullMergeOk := b.s.mo.FullMerge(k, ob, mergeOps)
if !fullMergeOk {
return nil, false
}
return mergedVal, true
})
}
for _, op := range b.ops {
if op.v != nil {
t.Set(op.k, op.v)
} else {
t.Delete(op.k)
}
}
return nil
}
func (b *Batch) Close() error {
return nil
}

112
vendor/github.com/blevesearch/bleve/index/store/cznicb/cznicb.go generated vendored
View file

@ -1,112 +0,0 @@
// Copyright (c) 2015 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
// +build go1.4
// Package cznicb provides an in-memory implementation of the KVStore
// interfaces using the cznic/b in-memory btree. Of note: this
// implementation does not have reader isolation.
package cznicb
import (
"bytes"
"fmt"
"sync"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/cznic/b"
)
const Name = "cznicb"
const MAX_CONCURRENT_WRITERS = 1
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
s := &Store{
t: b.TreeNew(itemCompare),
availableWriters: make(chan bool, MAX_CONCURRENT_WRITERS),
}
for i := 0; i < MAX_CONCURRENT_WRITERS; i++ {
s.availableWriters <- true
}
return s, nil
}
func itemCompare(a, b interface{}) int {
return bytes.Compare(a.([]byte), b.([]byte))
}
type Store struct {
availableWriters chan bool
m sync.RWMutex
t *b.Tree
mo store.MergeOperator
}
func (s *Store) Open() error {
return nil
}
func (s *Store) SetMergeOperator(mo store.MergeOperator) {
s.mo = mo
}
func (s *Store) Reader() (store.KVReader, error) {
return &Reader{s: s}, nil
}
func (s *Store) Writer() (store.KVWriter, error) {
available, ok := <-s.availableWriters
if !ok || !available {
return nil, fmt.Errorf("no available writers")
}
return &Writer{s: s, r: &Reader{s: s}}, nil
}
func (s *Store) Close() error {
return nil
}
func (s *Store) get(k []byte) ([]byte, error) {
s.m.RLock()
defer s.m.RUnlock()
v, ok := s.t.Get(k)
if !ok || v == nil {
return nil, nil
}
return v.([]byte), nil
}
func (s *Store) iterator(k []byte) store.KVIterator {
iter := &Iterator{s: s}
iter.Seek(k)
return iter
}
func (s *Store) set(k, v []byte) (err error) {
s.m.Lock()
defer s.m.Unlock()
s.t.Set(k, v)
return nil
}
func (s *Store) delete(k []byte) (err error) {
s.m.Lock()
defer s.m.Unlock()
s.t.Delete(k)
return nil
}

133
vendor/github.com/blevesearch/bleve/index/store/cznicb/cznicb_test.go generated vendored
View file

@ -1,133 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
package cznicb
import (
"testing"
"github.com/blevesearch/bleve/index/store"
)
func TestCznicBStore(t *testing.T) {
s, err := StoreConstructor(nil)
if err != nil {
t.Fatal(err)
}
CommonTestKVStore(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
v, err := writer.Get([]byte("a"))
if err != nil {
t.Fatal(err)
}
if string(v) != "val-a" {
t.Errorf("expected val-a")
}
v, err = writer.Get([]byte("not-there"))
if err != nil {
t.Fatal(err)
}
if v != nil {
t.Errorf("expected nil v")
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
}

113
vendor/github.com/blevesearch/bleve/index/store/cznicb/iterator.go generated vendored
View file

@ -1,113 +0,0 @@
// Copyright (c) 2015 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
// +build go1.4
package cznicb
import (
"errors"
"github.com/cznic/b"
)
var iteratorDoneErr = errors.New("iteratorDoneErr") // A sentinel value.
type Iterator struct { // Assuming that iterators are used single-threaded.
s *Store
e *b.Enumerator
currK interface{}
currV interface{}
currErr error
}
func (i *Iterator) SeekFirst() {
i.currK = nil
i.currV = nil
i.currErr = nil
var err error
i.s.m.RLock()
i.e, err = i.s.t.SeekFirst()
i.s.m.RUnlock() // cannot defer, must unlock before Next
if err != nil {
i.currK = nil
i.currV = nil
i.currErr = iteratorDoneErr
}
i.Next()
}
func (i *Iterator) Seek(k []byte) {
i.currK = nil
i.currV = nil
i.currErr = nil
i.s.m.RLock()
i.e, _ = i.s.t.Seek(k)
i.s.m.RUnlock() // cannot defer, must unlock before Next
i.Next()
}
func (i *Iterator) Next() {
if i.currErr != nil {
i.currK = nil
i.currV = nil
i.currErr = iteratorDoneErr
return
}
i.s.m.RLock()
defer i.s.m.RUnlock()
i.currK, i.currV, i.currErr = i.e.Next()
}
func (i *Iterator) Current() ([]byte, []byte, bool) {
if i.currErr == iteratorDoneErr ||
i.currK == nil ||
i.currV == nil {
return nil, nil, false
}
return i.currK.([]byte), i.currV.([]byte), true
}
func (i *Iterator) Key() []byte {
k, _, ok := i.Current()
if !ok {
return nil
}
return k
}
func (i *Iterator) Value() []byte {
_, v, ok := i.Current()
if !ok {
return nil
}
return v
}
func (i *Iterator) Valid() bool {
_, _, ok := i.Current()
return ok
}
func (i *Iterator) Close() error {
if i.e != nil {
i.e.Close()
}
i.e = nil
return nil
}

44
vendor/github.com/blevesearch/bleve/index/store/cznicb/reader.go generated vendored
View file

@ -1,44 +0,0 @@
// Copyright (c) 2015 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
// +build go1.4
package cznicb
import (
"github.com/blevesearch/bleve/index/store"
)
type Reader struct {
s *Store
}
func newReader(s *Store) (*Reader, error) {
return &Reader{
s: s,
}, nil
}
func (r *Reader) BytesSafeAfterClose() bool {
return false
}
func (r *Reader) Get(key []byte) ([]byte, error) {
return r.s.get(key)
}
func (r *Reader) Iterator(key []byte) store.KVIterator {
return r.s.iterator(key)
}
func (r *Reader) Close() error {
return nil
}

55
vendor/github.com/blevesearch/bleve/index/store/cznicb/writer.go generated vendored
View file

@ -1,55 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build go1.4
package cznicb
import (
"github.com/blevesearch/bleve/index/store"
)
type Writer struct {
s *Store
r *Reader
}
func (w *Writer) BytesSafeAfterClose() bool {
return false
}
func (w *Writer) Set(key, val []byte) error {
return w.s.set(key, val)
}
func (w *Writer) Delete(key []byte) error {
return w.s.delete(key)
}
func (w *Writer) NewBatch() store.KVBatch {
return &Batch{
s: w.s,
ops: make([]op, 0, 1000),
merges: make(map[string][][]byte),
}
}
func (w *Writer) Close() error {
w.s.availableWriters <- true
w.s = nil
return nil
}
func (w *Writer) Get(key []byte) ([]byte, error) {
return w.r.s.get(key)
}
func (w *Writer) Iterator(key []byte) store.KVIterator {
return w.r.s.iterator(key)
}

86
vendor/github.com/blevesearch/bleve/index/store/forestdb/batch.go generated vendored
View file

@ -1,86 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build forestdb
package forestdb
import (
"fmt"
)
type op struct {
k []byte
v []byte
}
type Batch struct {
s *Store
ops []op
merges map[string][][]byte
}
func (b *Batch) Set(k, v []byte) {
b.ops = append(b.ops, op{k, v})
}
func (b *Batch) Delete(k []byte) {
b.ops = append(b.ops, op{k, nil})
}
func (b *Batch) Merge(key, val []byte) {
ops, ok := b.merges[string(key)]
if ok && len(ops) > 0 {
last := ops[len(ops)-1]
mergedVal, partialMergeOk := b.s.mo.PartialMerge(key, last, val)
if partialMergeOk {
// replace last entry with the result of the merge
ops[len(ops)-1] = mergedVal
} else {
// could not partial merge, append this to the end
ops = append(ops, val)
}
} else {
ops = [][]byte{val}
}
b.merges[string(key)] = ops
}
func (b *Batch) Execute() (err error) {
for k, mergeOps := range b.merges {
kb := []byte(k)
existingVal, err := b.s.get(kb)
if err != nil {
return err
}
mergedVal, fullMergeOk := b.s.mo.FullMerge(kb, existingVal, mergeOps)
if !fullMergeOk {
return fmt.Errorf("merge operator returned failure")
}
err = b.s.setlocked(kb, mergedVal)
if err != nil {
return err
}
}
for _, op := range b.ops {
if op.v != nil {
b.s.setlocked(op.k, op.v)
} else {
b.s.deletelocked(op.k)
}
}
return b.s.commit()
}
func (b *Batch) Close() error {
return nil
}

117
vendor/github.com/blevesearch/bleve/index/store/forestdb/iterator.go generated vendored
View file

@ -1,117 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build forestdb
package forestdb
import (
"github.com/couchbase/goforestdb"
)
type Iterator struct {
store *Store
snapshot *forestdb.KVStore
iterator *forestdb.Iterator
curr *forestdb.Doc
valid bool
}
func newIterator(store *Store) *Iterator {
itr, err := store.dbkv.IteratorInit([]byte{}, nil, forestdb.ITR_NONE)
rv := Iterator{
store: store,
iterator: itr,
valid: err == nil,
}
return &rv
}
func newIteratorWithSnapshot(store *Store, snapshot *forestdb.KVStore) *Iterator {
itr, err := snapshot.IteratorInit([]byte{}, nil, forestdb.ITR_NONE)
rv := Iterator{
store: store,
iterator: itr,
valid: err == nil,
}
return &rv
}
func (i *Iterator) SeekFirst() {
err := i.iterator.SeekMin()
if err != nil {
i.valid = false
return
}
if i.curr != nil {
i.curr.Close()
}
i.curr, err = i.iterator.Get()
if err != nil {
i.valid = false
}
}
func (i *Iterator) Seek(key []byte) {
err := i.iterator.Seek(key, forestdb.FDB_ITR_SEEK_HIGHER)
if err != nil {
i.valid = false
return
}
if i.curr != nil {
i.curr.Close()
}
i.curr, err = i.iterator.Get()
if err != nil {
i.valid = false
return
}
}
func (i *Iterator) Next() {
err := i.iterator.Next()
if err != nil {
i.valid = false
return
}
if i.curr != nil {
i.curr.Close()
}
i.curr, err = i.iterator.Get()
if err != nil {
i.valid = false
}
}
func (i *Iterator) Current() ([]byte, []byte, bool) {
if i.Valid() {
return i.Key(), i.Value(), true
}
return nil, nil, false
}
func (i *Iterator) Key() []byte {
return i.curr.Key()
}
func (i *Iterator) Value() []byte {
return i.curr.Body()
}
func (i *Iterator) Valid() bool {
return i.valid
}
func (i *Iterator) Close() error {
i.valid = false
if i.curr != nil {
i.curr.Close()
}
return i.iterator.Close()
}

57
vendor/github.com/blevesearch/bleve/index/store/forestdb/reader.go generated vendored
View file

@ -1,57 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build forestdb
package forestdb
import (
"fmt"
"github.com/blevesearch/bleve/index/store"
"github.com/couchbase/goforestdb"
)
type Reader struct {
store *Store
snapshot *forestdb.KVStore
}
func (r *Reader) BytesSafeAfterClose() bool {
return true
}
func newReader(store *Store) (*Reader, error) {
snapshot, err := store.newSnapshot()
if err != nil {
return nil, fmt.Errorf("error opening snapshot: %v", err)
}
return &Reader{
store: store,
snapshot: snapshot,
}, nil
}
func (r *Reader) Get(key []byte) ([]byte, error) {
res, err := r.snapshot.GetKV(key)
if err != nil && err != forestdb.RESULT_KEY_NOT_FOUND {
return nil, err
}
return res, nil
}
func (r *Reader) Iterator(key []byte) store.KVIterator {
rv := newIteratorWithSnapshot(r.store, r.snapshot)
rv.Seek(key)
return rv
}
func (r *Reader) Close() error {
return r.snapshot.Close()
}

297
vendor/github.com/blevesearch/bleve/index/store/forestdb/store.go generated vendored
View file

@ -1,297 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build forestdb
package forestdb
import (
"bytes"
"encoding/binary"
"encoding/json"
"fmt"
"sync"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/couchbase/goforestdb"
)
type ForestDBConfig struct {
BlockSize uint32
BufferCacheSize uint64
ChunkSize uint16
CleanupCacheOnClose bool
CompactionBufferSizeMax uint32
CompactionMinimumFilesize uint64
CompactionMode forestdb.CompactOpt
CompactionThreshold uint8
CompactorSleepDuration uint64
CompressDocumentBody bool
DurabilityOpt forestdb.DurabilityOpt
OpenFlags forestdb.OpenFlags
PurgingInterval uint32
SeqTreeOpt forestdb.SeqTreeOpt
WalFlushBeforeCommit bool
WalThreshold uint64
}
const Name = "forestdb"
type Store struct {
path string
config *forestdb.Config
kvconfig *forestdb.KVStoreConfig
dbfile *forestdb.File
dbkv *forestdb.KVStore
writer sync.Mutex
mo store.MergeOperator
}
func New(path string, createIfMissing bool,
config map[string]interface{}) (*Store, error) {
if config == nil {
config = map[string]interface{}{}
}
forestDBDefaultConfig := forestdb.DefaultConfig()
forestDBDefaultConfig.SetCompactionMode(forestdb.COMPACT_AUTO)
forestDBConfig, err := applyConfig(forestDBDefaultConfig, config)
if err != nil {
return nil, err
}
rv := Store{
path: path,
config: forestDBConfig,
kvconfig: forestdb.DefaultKVStoreConfig(),
}
if createIfMissing {
rv.kvconfig.SetCreateIfMissing(true)
}
return &rv, nil
}
func (s *Store) Open() error {
var err error
s.dbfile, err = forestdb.Open(s.path, s.config)
if err != nil {
return err
}
s.dbkv, err = s.dbfile.OpenKVStoreDefault(s.kvconfig)
if err != nil {
return err
}
return nil
}
func (s *Store) SetMergeOperator(mo store.MergeOperator) {
s.mo = mo
}
func (s *Store) get(key []byte) ([]byte, error) {
res, err := s.dbkv.GetKV(key)
if err != nil && err != forestdb.RESULT_KEY_NOT_FOUND {
return nil, err
}
return res, nil
}
func (s *Store) set(key, val []byte) error {
s.writer.Lock()
defer s.writer.Unlock()
return s.setlocked(key, val)
}
func (s *Store) setlocked(key, val []byte) error {
return s.dbkv.SetKV(key, val)
}
func (s *Store) delete(key []byte) error {
s.writer.Lock()
defer s.writer.Unlock()
return s.deletelocked(key)
}
func (s *Store) deletelocked(key []byte) error {
return s.dbkv.DeleteKV(key)
}
func (s *Store) commit() error {
return s.dbfile.Commit(forestdb.COMMIT_NORMAL)
}
func (s *Store) Close() error {
err := s.dbkv.Close()
if err != nil {
return err
}
return s.dbfile.Close()
}
func (ldbs *Store) iterator(key []byte) store.KVIterator {
rv := newIterator(ldbs)
rv.Seek(key)
return rv
}
func (s *Store) Reader() (store.KVReader, error) {
return newReader(s)
}
func (ldbs *Store) Writer() (store.KVWriter, error) {
return newWriter(ldbs)
}
func (s *Store) getSeqNum() (forestdb.SeqNum, error) {
dbinfo, err := s.dbkv.Info()
if err != nil {
return 0, err
}
return dbinfo.LastSeqNum(), nil
}
func (s *Store) newSnapshot() (*forestdb.KVStore, error) {
seqNum, err := s.getSeqNum()
if err != nil {
return nil, fmt.Errorf("error getting snapshot seqnum: %v", err)
}
snapshot, err := s.dbkv.SnapshotOpen(seqNum)
if err == forestdb.RESULT_NO_DB_INSTANCE {
checkAgainSeqNum, err := s.getSeqNum()
if err != nil {
return nil, fmt.Errorf("error getting snapshot seqnum again: %v", err)
}
return nil, fmt.Errorf("cannot open snapshot %v, checked again its %v, error: %v", seqNum, checkAgainSeqNum, err)
}
return snapshot, err
}
func (s *Store) GetRollbackID() ([]byte, error) {
seqNum, err := s.getSeqNum()
if err != nil {
return nil, err
}
buf := new(bytes.Buffer)
err = binary.Write(buf, binary.LittleEndian, seqNum)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func (s *Store) RollbackTo(rollbackId []byte) error {
s.writer.Lock()
defer s.writer.Unlock()
buf := bytes.NewReader(rollbackId)
var seqNum forestdb.SeqNum
err := binary.Read(buf, binary.LittleEndian, &seqNum)
if err != nil {
return err
}
err = s.dbkv.Rollback(seqNum)
if err != nil {
return err
}
return nil
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
}
createIfMissing := false
cim, ok := config["create_if_missing"].(bool)
if ok {
createIfMissing = cim
}
return New(path, createIfMissing, config)
}
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
}
func applyConfig(c *forestdb.Config, config map[string]interface{}) (
*forestdb.Config, error) {
v, exists := config["forestDBConfig"]
if !exists || v == nil {
return c, nil
}
m, ok := v.(map[string]interface{})
if !ok {
return c, nil
}
// These extra steps of json.Marshal()/Unmarshal() help to convert
// to the types that we need for the setter calls.
b, err := json.Marshal(m)
if err != nil {
return nil, err
}
var f ForestDBConfig
err = json.Unmarshal(b, &f)
if err != nil {
return nil, err
}
if _, exists := m["blockSize"]; exists {
c.SetBlockSize(f.BlockSize)
}
if _, exists := m["bufferCacheSize"]; exists {
c.SetBufferCacheSize(f.BufferCacheSize)
}
if _, exists := m["chunkSize"]; exists {
c.SetChunkSize(f.ChunkSize)
}
if _, exists := m["cleanupCacheOnClose"]; exists {
c.SetCleanupCacheOnClose(f.CleanupCacheOnClose)
}
if _, exists := m["compactionBufferSizeMax"]; exists {
c.SetCompactionBufferSizeMax(f.CompactionBufferSizeMax)
}
if _, exists := m["compactionMinimumFilesize"]; exists {
c.SetCompactionMinimumFilesize(f.CompactionMinimumFilesize)
}
if _, exists := m["compactionMode"]; exists {
c.SetCompactionMode(f.CompactionMode)
}
if _, exists := m["compactionThreshold"]; exists {
c.SetCompactionThreshold(f.CompactionThreshold)
}
if _, exists := m["compactorSleepDuration"]; exists {
c.SetCompactorSleepDuration(f.CompactorSleepDuration)
}
if _, exists := m["compressDocumentBody"]; exists {
c.SetCompressDocumentBody(f.CompressDocumentBody)
}
if _, exists := m["durabilityOpt"]; exists {
c.SetDurabilityOpt(f.DurabilityOpt)
}
if _, exists := m["openFlags"]; exists {
c.SetOpenFlags(f.OpenFlags)
}
if _, exists := m["purgingInterval"]; exists {
c.SetPurgingInterval(f.PurgingInterval)
}
if _, exists := m["seqTreeOpt"]; exists {
c.SetSeqTreeOpt(f.SeqTreeOpt)
}
if _, exists := m["walFlushBeforeCommit"]; exists {
c.SetWalFlushBeforeCommit(f.WalFlushBeforeCommit)
}
if _, exists := m["walThreshold"]; exists {
c.SetWalThreshold(f.WalThreshold)
}
return c, nil
}

636
vendor/github.com/blevesearch/bleve/index/store/forestdb/store_test.go generated vendored
View file

@ -1,636 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build forestdb
package forestdb
import (
"os"
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
)
func TestForestDBStore(t *testing.T) {
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
t.Fatal(err)
}
}()
err := os.MkdirAll("testdir", 0700)
if err != nil {
t.Fatal(err)
}
s, err := New("testdir/test", true, nil)
if err != nil {
t.Fatal(err)
}
err = s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
CommonTestKVStore(t, s)
}
func TestReaderIsolation(t *testing.T) {
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
t.Fatal(err)
}
}()
err := os.MkdirAll("testdir", 0700)
if err != nil {
t.Fatal(err)
}
s, err := New("testdir/test", true, nil)
if err != nil {
t.Fatal(err)
}
err = s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
CommonTestReaderIsolation(t, s)
}
// TestRollbackSameHandle tries to rollback a handle
// and ensure that subsequent reads from it also
// reflect the rollback
func TestRollbackSameHandle(t *testing.T) {
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
t.Fatal(err)
}
}()
err := os.MkdirAll("testdir", 0700)
if err != nil {
t.Fatal(err)
}
s, err := New("testdir/test", true, nil)
if err != nil {
t.Fatal(err)
}
err = s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
writer, err := s.Writer()
if err != nil {
t.Fatal(err)
}
// create 2 docs, a and b
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("b"), []byte("val-b"))
if err != nil {
t.Error(err)
}
// get the rollback id
rollbackId, err := s.GetRollbackID()
if err != nil {
t.Error(err)
}
// create a 3rd doc c
err = writer.Set([]byte("c"), []byte("val-c"))
if err != nil {
t.Error(err)
}
err = writer.Close()
if err != nil {
t.Error(err)
}
// make sure c is there
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
val, err := reader.Get([]byte("c"))
if err != nil {
t.Error(err)
}
if string(val) != "val-c" {
t.Errorf("expected value 'val-c' got '%s'", val)
}
err = reader.Close()
if err != nil {
t.Fatal(err)
}
// now rollback
err = s.RollbackTo(rollbackId)
if err != nil {
t.Fatal(err)
}
// now make sure c is not there
reader, err = s.Reader()
if err != nil {
t.Error(err)
}
val, err = reader.Get([]byte("c"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected missing, got '%s'", val)
}
err = reader.Close()
if err != nil {
t.Fatal(err)
}
}
// TestRollbackNewHandle tries to rollback the
// database, then opens a new handle, and ensures
// that the rollback is reflected there as well
func TestRollbackNewHandle(t *testing.T) {
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
t.Fatal(err)
}
}()
err := os.MkdirAll("testdir", 0700)
if err != nil {
t.Fatal(err)
}
s, err := New("testdir/test", true, nil)
if err != nil {
t.Fatal(err)
}
err = s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
writer, err := s.Writer()
if err != nil {
t.Fatal(err)
}
// create 2 docs, a and b
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("b"), []byte("val-b"))
if err != nil {
t.Error(err)
}
// get the rollback id
rollbackId, err := s.GetRollbackID()
if err != nil {
t.Error(err)
}
// create a 3rd doc c
err = writer.Set([]byte("c"), []byte("val-c"))
if err != nil {
t.Error(err)
}
err = writer.Close()
if err != nil {
t.Error(err)
}
// make sure c is there
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
val, err := reader.Get([]byte("c"))
if err != nil {
t.Error(err)
}
if string(val) != "val-c" {
t.Errorf("expected value 'val-c' got '%s'", val)
}
err = reader.Close()
if err != nil {
t.Fatal(err)
}
// now rollback
err = s.RollbackTo(rollbackId)
if err != nil {
t.Fatal(err)
}
// now lets open another handle
s2, err := New("testdir/test", true, nil)
if err != nil {
t.Fatal(err)
}
err = s2.Open()
if err != nil {
t.Fatal(err)
}
defer s2.Close()
// now make sure c is not there
reader2, err := s2.Reader()
if err != nil {
t.Error(err)
}
val, err = reader2.Get([]byte("c"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected missing, got '%s'", val)
}
err = reader2.Close()
if err != nil {
t.Fatal(err)
}
}
// TestRollbackOtherHandle tries to create 2 handles
// at the beginning, then rollback one of them
// and ensure it affects the other
func TestRollbackOtherHandle(t *testing.T) {
defer func() {
err := os.RemoveAll("testdir")
if err != nil {
t.Fatal(err)
}
}()
err := os.MkdirAll("testdir", 0700)
if err != nil {
t.Fatal(err)
}
s, err := New("testdir/test", true, nil)
if err != nil {
t.Fatal(err)
}
err = s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
// open another handle at the same time
s2, err := New("testdir/test", true, nil)
if err != nil {
t.Fatal(err)
}
err = s2.Open()
if err != nil {
t.Fatal(err)
}
defer s2.Close()
writer, err := s.Writer()
if err != nil {
t.Fatal(err)
}
// create 2 docs, a and b
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("b"), []byte("val-b"))
if err != nil {
t.Error(err)
}
// get the rollback id
rollbackId, err := s.GetRollbackID()
if err != nil {
t.Error(err)
}
// create a 3rd doc c
err = writer.Set([]byte("c"), []byte("val-c"))
if err != nil {
t.Error(err)
}
err = writer.Close()
if err != nil {
t.Error(err)
}
// make sure c is there
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
val, err := reader.Get([]byte("c"))
if err != nil {
t.Error(err)
}
if string(val) != "val-c" {
t.Errorf("expected value 'val-c' got '%s'", val)
}
err = reader.Close()
if err != nil {
t.Fatal(err)
}
// now rollback
err = s.RollbackTo(rollbackId)
if err != nil {
t.Fatal(err)
}
// now make sure c is not on the other handle
reader2, err := s2.Reader()
if err != nil {
t.Error(err)
}
val, err = reader2.Get([]byte("c"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected missing, got '%s'", val)
}
err = reader2.Close()
if err != nil {
t.Fatal(err)
}
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
}
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
// insert a kv pair
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// create an isolated reader
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// verify we see the value already inserted
val, err := reader.Get([]byte("a"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-a")) {
t.Errorf("expected val-a, got nil")
}
// verify that an iterator sees it
count := 0
it := reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
// add something after the reader was created
writer, err = s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("b"), []byte("val-b"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// ensure that a newer reader sees it
newReader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := newReader.Close()
if err != nil {
t.Fatal(err)
}
}()
val, err = newReader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-b")) {
t.Errorf("expected val-b, got nil")
}
// ensure that the director iterator sees it
count = 0
it2 := newReader.Iterator([]byte{0})
defer func() {
err := it2.Close()
if err != nil {
t.Fatal(err)
}
}()
for it2.Valid() {
it2.Next()
count++
}
if count != 2 {
t.Errorf("expected iterator to see 2, saw %d", count)
}
// but that the isolated reader does not
val, err = reader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %v", val)
}
// and ensure that the iterator on the isolated reader also does not
count = 0
it3 := reader.Iterator([]byte{0})
defer func() {
err := it3.Close()
if err != nil {
t.Fatal(err)
}
}()
for it3.Valid() {
it3.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
}

71
vendor/github.com/blevesearch/bleve/index/store/forestdb/writer.go generated vendored
View file

@ -1,71 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build forestdb
package forestdb
import (
"github.com/blevesearch/bleve/index/store"
)
type Writer struct {
store *Store
}
func (w *Writer) BytesSafeAfterClose() bool {
return true
}
func newWriter(store *Store) (*Writer, error) {
store.writer.Lock()
return &Writer{
store: store,
}, nil
}
func (w *Writer) Set(key, val []byte) error {
err := w.store.setlocked(key, val)
if err != nil {
return err
}
return w.store.commit()
}
func (w *Writer) Delete(key []byte) error {
err := w.store.deletelocked(key)
if err != nil {
return err
}
return w.store.commit()
}
func (w *Writer) NewBatch() store.KVBatch {
return &Batch{
s: w.store,
ops: make([]op, 0, 1000),
merges: make(map[string][][]byte),
}
}
func (w *Writer) Close() error {
w.store.writer.Unlock()
return nil
}
// these two methods can safely read using the regular
// methods without a read transaction, because we know
// that no one else is writing but us
func (w *Writer) Get(key []byte) ([]byte, error) {
return w.store.get(key)
}
func (w *Writer) Iterator(key []byte) store.KVIterator {
return w.store.iterator(key)
}

53
vendor/github.com/blevesearch/bleve/index/store/goleveldb/batch.go generated vendored
View file

@ -1,53 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package goleveldb
import (
"github.com/blevesearch/bleve/index/store"
"github.com/syndtr/goleveldb/leveldb"
)
type Batch struct {
w *Writer
merge *store.EmulatedMerge
batch *leveldb.Batch
}
func (b *Batch) Set(key, val []byte) {
b.batch.Put(key, val)
}
func (b *Batch) Delete(key []byte) {
b.batch.Delete(key)
}
func (b *Batch) Merge(key, val []byte) {
b.merge.Merge(key, val)
}
func (b *Batch) Execute() error {
// first process merges
ops, err := b.merge.ExecuteDeferred(b.w)
if err != nil {
return err
}
for _, op := range ops {
b.batch.Put(op.K, op.V)
}
wopts := defaultWriteOptions()
err = b.w.store.db.Write(b.batch, wopts)
return err
}
func (b *Batch) Close() error {
return nil
}

75
vendor/github.com/blevesearch/bleve/index/store/goleveldb/iterator.go generated vendored
View file

@ -1,75 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package goleveldb
import (
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/iterator"
)
type Iterator struct {
store *Store
iterator iterator.Iterator
}
func newIterator(store *Store) *Iterator {
ropts := defaultReadOptions()
iter := store.db.NewIterator(nil, ropts)
rv := Iterator{
store: store,
iterator: iter,
}
return &rv
}
func newIteratorWithSnapshot(store *Store, snapshot *leveldb.Snapshot) *Iterator {
options := defaultReadOptions()
iter := snapshot.NewIterator(nil, options)
rv := Iterator{
store: store,
iterator: iter,
}
return &rv
}
func (ldi *Iterator) SeekFirst() {
ldi.iterator.First()
}
func (ldi *Iterator) Seek(key []byte) {
ldi.iterator.Seek(key)
}
func (ldi *Iterator) Next() {
ldi.iterator.Next()
}
func (ldi *Iterator) Current() ([]byte, []byte, bool) {
if ldi.Valid() {
return ldi.Key(), ldi.Value(), true
}
return nil, nil, false
}
func (ldi *Iterator) Key() []byte {
return ldi.iterator.Key()
}
func (ldi *Iterator) Value() []byte {
return ldi.iterator.Value()
}
func (ldi *Iterator) Valid() bool {
return ldi.iterator.Valid()
}
func (ldi *Iterator) Close() error {
return nil
}

47
vendor/github.com/blevesearch/bleve/index/store/goleveldb/reader.go generated vendored
View file

@ -1,47 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package goleveldb
import (
"github.com/blevesearch/bleve/index/store"
"github.com/syndtr/goleveldb/leveldb"
)
type Reader struct {
store *Store
snapshot *leveldb.Snapshot
}
func newReader(store *Store) (*Reader, error) {
snapshot, _ := store.db.GetSnapshot()
return &Reader{
store: store,
snapshot: snapshot,
}, nil
}
func (r *Reader) BytesSafeAfterClose() bool {
return false
}
func (r *Reader) Get(key []byte) ([]byte, error) {
return r.store.getWithSnapshot(key, r.snapshot)
}
func (r *Reader) Iterator(key []byte) store.KVIterator {
rv := newIteratorWithSnapshot(r.store, r.snapshot)
rv.Seek(key)
return rv
}
func (r *Reader) Close() error {
r.snapshot.Release()
return nil
}

172
vendor/github.com/blevesearch/bleve/index/store/goleveldb/store.go generated vendored
View file

@ -1,172 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package goleveldb
import (
"fmt"
"sync"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/filter"
"github.com/syndtr/goleveldb/leveldb/opt"
)
const Name = "goleveldb"
type Store struct {
path string
opts *opt.Options
db *leveldb.DB
writer sync.Mutex
mo store.MergeOperator
}
func New(path string, config map[string]interface{}) (*Store, error) {
rv := Store{
path: path,
opts: &opt.Options{},
}
_, err := applyConfig(rv.opts, config)
if err != nil {
return nil, err
}
return &rv, nil
}
func (ldbs *Store) Open() error {
var err error
ldbs.db, err = leveldb.OpenFile(ldbs.path, ldbs.opts)
if err != nil {
return err
}
return nil
}
func (ldbs *Store) SetMergeOperator(mo store.MergeOperator) {
ldbs.mo = mo
}
func (ldbs *Store) get(key []byte) ([]byte, error) {
options := defaultReadOptions()
b, err := ldbs.db.Get(key, options)
if err == leveldb.ErrNotFound {
return nil, nil
}
return b, err
}
func (ldbs *Store) getWithSnapshot(key []byte, snapshot *leveldb.Snapshot) ([]byte, error) {
options := defaultReadOptions()
b, err := snapshot.Get(key, options)
if err == leveldb.ErrNotFound {
return nil, nil
}
return b, err
}
func (ldbs *Store) set(key, val []byte) error {
ldbs.writer.Lock()
defer ldbs.writer.Unlock()
return ldbs.setlocked(key, val)
}
func (ldbs *Store) setlocked(key, val []byte) error {
options := defaultWriteOptions()
err := ldbs.db.Put(key, val, options)
return err
}
func (ldbs *Store) delete(key []byte) error {
ldbs.writer.Lock()
defer ldbs.writer.Unlock()
return ldbs.deletelocked(key)
}
func (ldbs *Store) deletelocked(key []byte) error {
options := defaultWriteOptions()
err := ldbs.db.Delete(key, options)
return err
}
func (ldbs *Store) Close() error {
return ldbs.db.Close()
}
func (ldbs *Store) iterator(key []byte) store.KVIterator {
rv := newIterator(ldbs)
rv.Seek(key)
return rv
}
func (ldbs *Store) Reader() (store.KVReader, error) {
return newReader(ldbs)
}
func (ldbs *Store) Writer() (store.KVWriter, error) {
return newWriter(ldbs)
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
}
return New(path, config)
}
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
}
func applyConfig(o *opt.Options, config map[string]interface{}) (
*opt.Options, error) {
cim, ok := config["create_if_missing"].(bool)
if ok {
o.ErrorIfMissing = !cim
}
eie, ok := config["error_if_exists"].(bool)
if ok {
o.ErrorIfExist = eie
}
wbs, ok := config["write_buffer_size"].(float64)
if ok {
o.WriteBuffer = int(wbs)
}
bs, ok := config["block_size"].(float64)
if ok {
o.BlockSize = int(bs)
}
bri, ok := config["block_restart_interval"].(float64)
if ok {
o.BlockRestartInterval = int(bri)
}
lcc, ok := config["lru_cache_capacity"].(float64)
if ok {
o.BlockCacheCapacity = int(lcc)
}
bfbpk, ok := config["bloom_filter_bits_per_key"].(float64)
if ok {
bf := filter.NewBloomFilter(int(bfbpk))
o.Filter = bf
}
return o, nil
}

296
vendor/github.com/blevesearch/bleve/index/store/goleveldb/store_test.go generated vendored
View file

@ -1,296 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package goleveldb
import (
"os"
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
)
var leveldbTestOptions = map[string]interface{}{
"create_if_missing": true,
}
func TestLevelDBStore(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
s, err := New("test", leveldbTestOptions)
if err != nil {
t.Fatal(err)
}
err = s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
CommonTestKVStore(t, s)
}
func TestReaderIsolation(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
s, err := New("test", leveldbTestOptions)
if err != nil {
t.Fatal(err)
}
err = s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
CommonTestReaderIsolation(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
}
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
// insert a kv pair
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// create an isolated reader
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// verify that we see the value already inserted
val, err := reader.Get([]byte("a"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-a")) {
t.Errorf("expected val-a, got nil")
}
// verify that an iterator sees it
count := 0
it := reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
// add something after the reader was created
writer, err = s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("b"), []byte("val-b"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// ensure that a newer reader sees it
newReader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := newReader.Close()
if err != nil {
t.Fatal(err)
}
}()
val, err = newReader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-b")) {
t.Errorf("expected val-b, got nil")
}
// ensure that the director iterator sees it
count = 0
it = newReader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 2 {
t.Errorf("expected iterator to see 2, saw %d", count)
}
// but that the isolated reader does not
val, err = reader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %v", val)
}
// and ensure that the iterator on the isolated reader also does not
count = 0
it = reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
}

26
vendor/github.com/blevesearch/bleve/index/store/goleveldb/util.go generated vendored
View file

@ -1,26 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package goleveldb
import (
"github.com/syndtr/goleveldb/leveldb/opt"
)
func defaultWriteOptions() *opt.WriteOptions {
wo := &opt.WriteOptions{}
// request fsync on write for safety
wo.Sync = true
return wo
}
func defaultReadOptions() *opt.ReadOptions {
ro := &opt.ReadOptions{}
return ro
}

63
vendor/github.com/blevesearch/bleve/index/store/goleveldb/writer.go generated vendored
View file

@ -1,63 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package goleveldb
import (
"github.com/blevesearch/bleve/index/store"
"github.com/syndtr/goleveldb/leveldb"
)
type Writer struct {
store *Store
}
func newWriter(store *Store) (*Writer, error) {
store.writer.Lock()
return &Writer{
store: store,
}, nil
}
func (w *Writer) BytesSafeAfterClose() bool {
return false
}
func (w *Writer) Set(key, val []byte) error {
return w.store.setlocked(key, val)
}
func (w *Writer) Delete(key []byte) error {
return w.store.deletelocked(key)
}
func (w *Writer) NewBatch() store.KVBatch {
rv := Batch{
w: w,
merge: store.NewEmulatedMerge(w.store.mo),
batch: new(leveldb.Batch),
}
return &rv
}
func (w *Writer) Close() error {
w.store.writer.Unlock()
return nil
}
// these two methods can safely read using the regular
// methods without a read transaction, because we know
// that no one else is writing but us
func (w *Writer) Get(key []byte) ([]byte, error) {
return w.store.get(key)
}
func (w *Writer) Iterator(key []byte) store.KVIterator {
return w.store.iterator(key)
}

43
vendor/github.com/blevesearch/bleve/index/store/gorocksdb/batch.go generated vendored
View file

@ -1,43 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build rocksdb
package rocksdb
import (
"github.com/tecbot/gorocksdb"
)
type Batch struct {
w *Writer
batch *gorocksdb.WriteBatch
}
func (b *Batch) Set(key, val []byte) {
b.batch.Put(key, val)
}
func (b *Batch) Delete(key []byte) {
b.batch.Delete(key)
}
func (b *Batch) Merge(key, val []byte) {
b.batch.Merge(key, val)
}
func (b *Batch) Execute() error {
wopts := defaultWriteOptions()
err := b.w.store.db.Write(wopts, b.batch)
return err
}
func (b *Batch) Close() error {
return nil
}

76
vendor/github.com/blevesearch/bleve/index/store/gorocksdb/iterator.go generated vendored
View file

@ -1,76 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build rocksdb
package rocksdb
import (
"github.com/tecbot/gorocksdb"
)
type Iterator struct {
store *Store
iterator *gorocksdb.Iterator
}
func newIterator(store *Store) *Iterator {
ropts := defaultReadOptions()
rv := Iterator{
store: store,
iterator: store.db.NewIterator(ropts),
}
return &rv
}
func newIteratorWithSnapshot(store *Store, snapshot *gorocksdb.Snapshot) *Iterator {
options := defaultReadOptions()
options.SetSnapshot(snapshot)
rv := Iterator{
store: store,
iterator: store.db.NewIterator(options),
}
return &rv
}
func (ldi *Iterator) SeekFirst() {
ldi.iterator.SeekToFirst()
}
func (ldi *Iterator) Seek(key []byte) {
ldi.iterator.Seek(key)
}
func (ldi *Iterator) Next() {
ldi.iterator.Next()
}
func (ldi *Iterator) Current() ([]byte, []byte, bool) {
if ldi.Valid() {
return ldi.Key(), ldi.Value(), true
}
return nil, nil, false
}
func (ldi *Iterator) Key() []byte {
return ldi.iterator.Key().Data()
}
func (ldi *Iterator) Value() []byte {
return ldi.iterator.Value().Data()
}
func (ldi *Iterator) Valid() bool {
return ldi.iterator.Valid()
}
func (ldi *Iterator) Close() error {
ldi.iterator.Close()
return nil
}

48
vendor/github.com/blevesearch/bleve/index/store/gorocksdb/reader.go generated vendored
View file

@ -1,48 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build rocksdb
package rocksdb
import (
"github.com/blevesearch/bleve/index/store"
"github.com/tecbot/gorocksdb"
)
type Reader struct {
store *Store
snapshot *gorocksdb.Snapshot
}
func newReader(store *Store) (*Reader, error) {
return &Reader{
store: store,
snapshot: store.db.NewSnapshot(),
}, nil
}
func (r *Reader) BytesSafeAfterClose() bool {
return false
}
func (r *Reader) Get(key []byte) ([]byte, error) {
return r.store.getWithSnapshot(key, r.snapshot)
}
func (r *Reader) Iterator(key []byte) store.KVIterator {
rv := newIteratorWithSnapshot(r.store, r.snapshot)
rv.Seek(key)
return rv
}
func (r *Reader) Close() error {
r.snapshot.Release()
return nil
}

146
vendor/github.com/blevesearch/bleve/index/store/gorocksdb/store.go generated vendored
View file

@ -1,146 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build rocksdb
package rocksdb
import (
"fmt"
"sync"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/tecbot/gorocksdb"
)
const Name = "rocksdb"
type Store struct {
path string
opts *gorocksdb.Options
db *gorocksdb.DB
writer sync.Mutex
}
func New(path string, config map[string]interface{}) (*Store, error) {
rv := Store{
path: path,
opts: gorocksdb.NewDefaultOptions(),
}
_, err := applyConfig(rv.opts, config)
if err != nil {
return nil, err
}
return &rv, nil
}
func (ldbs *Store) Open() error {
var err error
ldbs.db, err = gorocksdb.OpenDb(ldbs.opts, ldbs.path)
if err != nil {
return err
}
return nil
}
func (ldbs *Store) SetMergeOperator(mo store.MergeOperator) {
ldbs.opts.SetMergeOperator(mo)
}
func (ldbs *Store) get(key []byte) ([]byte, error) {
options := defaultReadOptions()
b, err := ldbs.db.Get(options, key)
return b.Data(), err
}
func (ldbs *Store) getWithSnapshot(key []byte, snapshot *gorocksdb.Snapshot) ([]byte, error) {
options := defaultReadOptions()
options.SetSnapshot(snapshot)
b, err := ldbs.db.Get(options, key)
return b.Data(), err
}
func (ldbs *Store) set(key, val []byte) error {
ldbs.writer.Lock()
defer ldbs.writer.Unlock()
return ldbs.setlocked(key, val)
}
func (ldbs *Store) setlocked(key, val []byte) error {
options := defaultWriteOptions()
err := ldbs.db.Put(options, key, val)
return err
}
func (ldbs *Store) delete(key []byte) error {
ldbs.writer.Lock()
defer ldbs.writer.Unlock()
return ldbs.deletelocked(key)
}
func (ldbs *Store) deletelocked(key []byte) error {
options := defaultWriteOptions()
err := ldbs.db.Delete(options, key)
return err
}
func (ldbs *Store) Close() error {
ldbs.db.Close()
return nil
}
func (ldbs *Store) iterator(key []byte) store.KVIterator {
rv := newIterator(ldbs)
rv.Seek(key)
return rv
}
func (ldbs *Store) Reader() (store.KVReader, error) {
return newReader(ldbs)
}
func (ldbs *Store) Writer() (store.KVWriter, error) {
return newWriter(ldbs)
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
}
return New(path, config)
}
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
}
func applyConfig(o *gorocksdb.Options, config map[string]interface{}) (
*gorocksdb.Options, error) {
cim, ok := config["create_if_missing"].(bool)
if ok {
o.SetCreateIfMissing(cim)
}
eie, ok := config["error_if_exists"].(bool)
if ok {
o.SetErrorIfExists(eie)
}
wbs, ok := config["write_buffer_size"].(float64)
if ok {
o.SetWriteBufferSize(int(wbs))
}
return o, nil
}

298
vendor/github.com/blevesearch/bleve/index/store/gorocksdb/store_test.go generated vendored
View file

@ -1,298 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build rocksdb
package rocksdb
import (
"os"
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
)
var rocksdbTestOptions = map[string]interface{}{
"create_if_missing": true,
}
func TestGoRocksDBStore(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
s, err := New("test", rocksdbTestOptions)
if err != nil {
t.Fatal(err)
}
err = s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
CommonTestKVStore(t, s)
}
func TestReaderIsolation(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
s, err := New("test", rocksdbTestOptions)
if err != nil {
t.Fatal(err)
}
err = s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
CommonTestReaderIsolation(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
}
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
// insert a kv pair
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// create an isolated reader
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// verify that we see the value already inserted
val, err := reader.Get([]byte("a"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-a")) {
t.Errorf("expected val-a, got nil")
}
// verify that an iterator sees it
count := 0
it := reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
// add something after the reader was created
writer, err = s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("b"), []byte("val-b"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// ensure that a newer reader sees it
newReader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := newReader.Close()
if err != nil {
t.Fatal(err)
}
}()
val, err = newReader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-b")) {
t.Errorf("expected val-b, got nil")
}
// ensure that the director iterator sees it
count = 0
it2 := newReader.Iterator([]byte{0})
defer func() {
err := it2.Close()
if err != nil {
t.Fatal(err)
}
}()
for it2.Valid() {
it2.Next()
count++
}
if count != 2 {
t.Errorf("expected iterator to see 2, saw %d", count)
}
// but that the isolated reader does not
val, err = reader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %v", val)
}
// and ensure that the iterator on the isolated reader also does not
count = 0
it3 := reader.Iterator([]byte{0})
defer func() {
err := it3.Close()
if err != nil {
t.Fatal(err)
}
}()
for it3.Valid() {
it3.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
}

28
vendor/github.com/blevesearch/bleve/index/store/gorocksdb/util.go generated vendored
View file

@ -1,28 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build rocksdb
package rocksdb
import (
"github.com/tecbot/gorocksdb"
)
func defaultWriteOptions() *gorocksdb.WriteOptions {
wo := gorocksdb.NewDefaultWriteOptions()
// request fsync on write for safety
wo.SetSync(true)
return wo
}
func defaultReadOptions() *gorocksdb.ReadOptions {
ro := gorocksdb.NewDefaultReadOptions()
return ro
}

64
vendor/github.com/blevesearch/bleve/index/store/gorocksdb/writer.go generated vendored
View file

@ -1,64 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build rocksdb
package rocksdb
import (
"github.com/blevesearch/bleve/index/store"
"github.com/tecbot/gorocksdb"
)
type Writer struct {
store *Store
}
func newWriter(store *Store) (*Writer, error) {
store.writer.Lock()
return &Writer{
store: store,
}, nil
}
func (w *Writer) BytesSafeAfterClose() bool {
return false
}
func (w *Writer) Set(key, val []byte) error {
return w.store.setlocked(key, val)
}
func (w *Writer) Delete(key []byte) error {
return w.store.deletelocked(key)
}
func (w *Writer) NewBatch() store.KVBatch {
rv := Batch{
w: w,
batch: gorocksdb.NewWriteBatch(),
}
return &rv
}
func (w *Writer) Close() error {
w.store.writer.Unlock()
return nil
}
// these two methods can safely read using the regular
// methods without a read transaction, because we know
// that no one else is writing but us
func (w *Writer) Get(key []byte) ([]byte, error) {
return w.store.get(key)
}
func (w *Writer) Iterator(key []byte) store.KVIterator {
return w.store.iterator(key)
}

96
vendor/github.com/blevesearch/bleve/index/store/gtreap/gtreap.go generated vendored
View file

@ -1,96 +0,0 @@
// Copyright (c) 2015 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
// Package gtreap provides an in-memory implementation of the
// KVStore interfaces using the gtreap balanced-binary treap,
// copy-on-write data structure.
package gtreap
import (
"bytes"
"fmt"
"sync"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/steveyen/gtreap"
)
const Name = "gtreap"
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
}
const MAX_CONCURRENT_WRITERS = 1
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
s := &Store{
availableWriters: make(chan bool, MAX_CONCURRENT_WRITERS),
t: gtreap.NewTreap(itemCompare),
}
for i := 0; i < MAX_CONCURRENT_WRITERS; i++ {
s.availableWriters <- true
}
return s, nil
}
type Item struct {
k []byte
v []byte
}
func itemCompare(a, b interface{}) int {
return bytes.Compare(a.(*Item).k, b.(*Item).k)
}
type Store struct {
availableWriters chan bool
m sync.Mutex
t *gtreap.Treap
mo store.MergeOperator
}
type Writer struct {
s *Store
}
func (s *Store) Open() error {
return nil
}
func (s *Store) SetMergeOperator(mo store.MergeOperator) {
s.mo = mo
}
func (s *Store) Close() error {
close(s.availableWriters)
return nil
}
func (s *Store) Reader() (store.KVReader, error) {
s.m.Lock()
t := s.t
s.m.Unlock()
return &Reader{t: t}, nil
}
func (s *Store) Writer() (store.KVWriter, error) {
available, ok := <-s.availableWriters
if !ok || !available {
return nil, fmt.Errorf("no available writers")
}
return &Writer{s: s}, nil
}

259
vendor/github.com/blevesearch/bleve/index/store/gtreap/gtreap_test.go generated vendored
View file

@ -1,259 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
package gtreap
import (
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
)
func TestGTreapStore(t *testing.T) {
s, err := StoreConstructor(nil)
if err != nil {
t.Fatal(err)
}
CommonTestKVStore(t, s)
}
func TestReaderIsolation(t *testing.T) {
s, err := StoreConstructor(nil)
if err != nil {
t.Fatal(err)
}
CommonTestReaderIsolation(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
}
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
// insert a kv pair
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// create an isolated reader
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// verify that we see the value already inserted
val, err := reader.Get([]byte("a"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-a")) {
t.Errorf("expected val-a, got nil")
}
// verify that an iterator sees it
count := 0
it := reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
// add something after the reader was created
writer, err = s.Writer()
if err != nil {
t.Error(err)
}
valB := []byte("val-b")
err = writer.Set([]byte("b"), valB)
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// ensure that a newer reader sees it
newReader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := newReader.Close()
if err != nil {
t.Fatal(err)
}
}()
val, err = newReader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-b")) {
t.Errorf("expected val-b, got %s", val)
}
// ensure that the director iterator sees it
count = 0
it2 := newReader.Iterator([]byte{0})
defer func() {
err := it2.Close()
if err != nil {
t.Fatal(err)
}
}()
for it2.Valid() {
it2.Next()
count++
}
if count != 2 {
t.Errorf("expected iterator to see 2, saw %d", count)
}
// but that the isolated reader does not
val, err = reader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %v", val)
}
// and ensure that the iterator on the isolated reader also does not
count = 0
it3 := reader.Iterator([]byte{0})
defer func() {
err := it3.Close()
if err != nil {
t.Fatal(err)
}
}()
for it3.Valid() {
it3.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
}

62
vendor/github.com/blevesearch/bleve/index/store/gtreap/iterator.go generated vendored
View file

@ -1,13 +1,16 @@
// Copyright (c) 2015 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package gtreap provides an in-memory implementation of the
// KVStore interfaces using the gtreap balanced-binary treap,
@ -15,6 +18,7 @@
package gtreap
import (
"bytes"
"sync"
"github.com/steveyen/gtreap"
@ -28,22 +32,33 @@ type Iterator struct {
nextCh chan *Item
curr *Item
currOk bool
}
func newIterator(t *gtreap.Treap) *Iterator {
return &Iterator{t: t}
}
func (w *Iterator) SeekFirst() {
min := w.t.Min()
if min != nil {
w.restart(min.(*Item))
} else {
w.restart(nil)
}
prefix []byte
start []byte
end []byte
}
func (w *Iterator) Seek(k []byte) {
if w.start != nil && bytes.Compare(k, w.start) < 0 {
k = w.start
}
if w.prefix != nil && !bytes.HasPrefix(k, w.prefix) {
if bytes.Compare(k, w.prefix) < 0 {
k = w.prefix
} else {
var end []byte
for i := len(w.prefix) - 1; i >= 0; i-- {
c := w.prefix[i]
if c < 0xff {
end = make([]byte, i+1)
copy(end, w.prefix)
end[i] = c + 1
break
}
}
k = end
}
}
w.restart(&Item{k: k})
}
@ -93,6 +108,11 @@ func (w *Iterator) Current() ([]byte, []byte, bool) {
if !w.currOk || w.curr == nil {
return nil, nil, false
}
if w.prefix != nil && !bytes.HasPrefix(w.curr.k, w.prefix) {
return nil, nil, false
} else if w.end != nil && bytes.Compare(w.curr.k, w.end) >= 0 {
return nil, nil, false
}
return w.curr.k, w.curr.v, w.currOk
}

53
vendor/github.com/blevesearch/bleve/index/store/gtreap/reader.go generated vendored
View file

@ -1,13 +1,16 @@
// Copyright (c) 2015 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package gtreap provides an in-memory implementation of the
// KVStore interfaces using the gtreap balanced-binary treap,
@ -24,20 +27,38 @@ type Reader struct {
t *gtreap.Treap
}
func (w *Reader) BytesSafeAfterClose() bool {
return false
}
func (w *Reader) Get(k []byte) (v []byte, err error) {
var rv []byte
itm := w.t.Get(&Item{k: k})
if itm != nil {
return itm.(*Item).v, nil
rv = make([]byte, len(itm.(*Item).v))
copy(rv, itm.(*Item).v)
return rv, nil
}
return nil, nil
}
func (w *Reader) Iterator(k []byte) store.KVIterator {
return newIterator(w.t).restart(&Item{k: k})
func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {
return store.MultiGet(r, keys)
}
func (w *Reader) PrefixIterator(k []byte) store.KVIterator {
rv := Iterator{
t: w.t,
prefix: k,
}
rv.restart(&Item{k: k})
return &rv
}
func (w *Reader) RangeIterator(start, end []byte) store.KVIterator {
rv := Iterator{
t: w.t,
start: start,
end: end,
}
rv.restart(&Item{k: start})
return &rv
}
func (w *Reader) Close() error {

82
vendor/github.com/blevesearch/bleve/index/store/gtreap/store.go generated vendored Normal file
View file

@ -0,0 +1,82 @@
// Copyright (c) 2015 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package gtreap provides an in-memory implementation of the
// KVStore interfaces using the gtreap balanced-binary treap,
// copy-on-write data structure.
package gtreap
import (
"bytes"
"fmt"
"os"
"sync"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/steveyen/gtreap"
)
const Name = "gtreap"
type Store struct {
m sync.Mutex
t *gtreap.Treap
mo store.MergeOperator
}
type Item struct {
k []byte
v []byte
}
func itemCompare(a, b interface{}) int {
return bytes.Compare(a.(*Item).k, b.(*Item).k)
}
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
}
if path != "" {
return nil, os.ErrInvalid
}
rv := Store{
t: gtreap.NewTreap(itemCompare),
mo: mo,
}
return &rv, nil
}
func (s *Store) Close() error {
return nil
}
func (s *Store) Reader() (store.KVReader, error) {
s.m.Lock()
t := s.t
s.m.Unlock()
return &Reader{t: t}, nil
}
func (s *Store) Writer() (store.KVWriter, error) {
return &Writer{s: s}, nil
}
func init() {
registry.RegisterKVStore(Name, New)
}

114
vendor/github.com/blevesearch/bleve/index/store/gtreap/writer.go generated vendored
View file

@ -1,13 +1,16 @@
// Copyright (c) 2015 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package gtreap provides an in-memory implementation of the
// KVStore interfaces using the gtreap balanced-binary treap,
@ -15,58 +18,59 @@
package gtreap
import (
"fmt"
"math/rand"
"github.com/blevesearch/bleve/index/store"
)
func (w *Writer) BytesSafeAfterClose() bool {
return false
}
func (w *Writer) Get(k []byte) (v []byte, err error) {
w.s.m.Lock()
t := w.s.t
w.s.m.Unlock()
itm := t.Get(&Item{k: k})
if itm != nil {
return itm.(*Item).v, nil
}
return nil, nil
}
func (w *Writer) Iterator(k []byte) store.KVIterator {
w.s.m.Lock()
t := w.s.t
w.s.m.Unlock()
return newIterator(t).restart(&Item{k: k})
}
func (w *Writer) Close() error {
w.s.availableWriters <- true
w.s = nil
return nil
}
func (w *Writer) Set(k, v []byte) (err error) {
w.s.m.Lock()
w.s.t = w.s.t.Upsert(&Item{k: k, v: v}, rand.Int())
w.s.m.Unlock()
return nil
}
func (w *Writer) Delete(k []byte) (err error) {
w.s.m.Lock()
w.s.t = w.s.t.Delete(&Item{k: k})
w.s.m.Unlock()
return nil
type Writer struct {
s *Store
}
func (w *Writer) NewBatch() store.KVBatch {
return store.NewEmulatedBatch(w, w.s.mo)
return store.NewEmulatedBatch(w.s.mo)
}
func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) {
return make([]byte, options.TotalBytes), w.NewBatch(), nil
}
func (w *Writer) ExecuteBatch(batch store.KVBatch) error {
emulatedBatch, ok := batch.(*store.EmulatedBatch)
if !ok {
return fmt.Errorf("wrong type of batch")
}
w.s.m.Lock()
for k, mergeOps := range emulatedBatch.Merger.Merges {
kb := []byte(k)
var existingVal []byte
existingItem := w.s.t.Get(&Item{k: kb})
if existingItem != nil {
existingVal = w.s.t.Get(&Item{k: kb}).(*Item).v
}
mergedVal, fullMergeOk := w.s.mo.FullMerge(kb, existingVal, mergeOps)
if !fullMergeOk {
return fmt.Errorf("merge operator returned failure")
}
w.s.t = w.s.t.Upsert(&Item{k: kb, v: mergedVal}, rand.Int())
}
for _, op := range emulatedBatch.Ops {
if op.V != nil {
w.s.t = w.s.t.Upsert(&Item{k: op.K, v: op.V}, rand.Int())
} else {
w.s.t = w.s.t.Delete(&Item{k: op.K})
}
}
w.s.m.Unlock()
return nil
}
func (w *Writer) Close() error {
w.s = nil
return nil
}

70
vendor/github.com/blevesearch/bleve/index/store/inmem/iterator.go generated vendored
View file

@ -1,70 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package inmem
import (
"github.com/ryszard/goskiplist/skiplist"
)
type Iterator struct {
store *Store
iterator skiplist.Iterator
valid bool
}
func newIterator(store *Store) *Iterator {
rv := Iterator{
store: store,
iterator: store.list.Iterator(),
}
return &rv
}
func (i *Iterator) SeekFirst() {
i.Seek([]byte{0})
}
func (i *Iterator) Seek(k []byte) {
i.valid = i.iterator.Seek(string(k))
}
func (i *Iterator) Next() {
i.valid = i.iterator.Next()
}
func (i *Iterator) Current() ([]byte, []byte, bool) {
if i.valid {
return []byte(i.Key()), []byte(i.Value()), true
}
return nil, nil, false
}
func (i *Iterator) Key() []byte {
if i.valid {
return []byte(i.iterator.Key().(string))
}
return nil
}
func (i *Iterator) Value() []byte {
if i.valid {
return []byte(i.iterator.Value().(string))
}
return nil
}
func (i *Iterator) Valid() bool {
return i.valid
}
func (i *Iterator) Close() error {
i.iterator.Close()
return nil
}

40
vendor/github.com/blevesearch/bleve/index/store/inmem/reader.go generated vendored
View file

@ -1,40 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package inmem
import (
"github.com/blevesearch/bleve/index/store"
)
type Reader struct {
store *Store
}
func newReader(store *Store) (*Reader, error) {
return &Reader{
store: store,
}, nil
}
func (r *Reader) BytesSafeAfterClose() bool {
return false
}
func (r *Reader) Get(key []byte) ([]byte, error) {
return r.store.get(key)
}
func (r *Reader) Iterator(key []byte) store.KVIterator {
return r.store.iterator(key)
}
func (r *Reader) Close() error {
return nil
}

106
vendor/github.com/blevesearch/bleve/index/store/inmem/store.go generated vendored
View file

@ -1,106 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package inmem
import (
"sync"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/ryszard/goskiplist/skiplist"
)
const Name = "mem"
type Store struct {
list *skiplist.SkipList
writer sync.Mutex
mo store.MergeOperator
}
func New() (*Store, error) {
rv := Store{
list: skiplist.NewStringMap(),
}
return &rv, nil
}
func MustOpen() *Store {
rv := Store{
list: skiplist.NewStringMap(),
}
return &rv
}
func (i *Store) Open() error {
return nil
}
func (i *Store) SetMergeOperator(mo store.MergeOperator) {
i.mo = mo
}
func (i *Store) get(key []byte) ([]byte, error) {
val, ok := i.list.Get(string(key))
if ok {
return []byte(val.(string)), nil
}
return nil, nil
}
func (i *Store) set(key, val []byte) error {
i.writer.Lock()
defer i.writer.Unlock()
return i.setlocked(key, val)
}
func (i *Store) setlocked(key, val []byte) error {
i.list.Set(string(key), string(val))
return nil
}
func (i *Store) delete(key []byte) error {
i.writer.Lock()
defer i.writer.Unlock()
return i.deletelocked(key)
}
func (i *Store) deletelocked(key []byte) error {
i.list.Delete(string(key))
return nil
}
func (i *Store) Close() error {
return nil
}
func (i *Store) iterator(key []byte) store.KVIterator {
rv := newIterator(i)
rv.Seek(key)
return rv
}
func (i *Store) Reader() (store.KVReader, error) {
return newReader(i)
}
func (i *Store) Writer() (store.KVWriter, error) {
return newWriter(i)
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
return New()
}
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
}

254
vendor/github.com/blevesearch/bleve/index/store/inmem/store_test.go generated vendored
View file

@ -1,254 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package inmem
import (
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
)
func TestStore(t *testing.T) {
s, err := New()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
CommonTestKVStore(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
}
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
// insert a kv pair
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// create an isolated reader
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// verify that we see the value already inserted
val, err := reader.Get([]byte("a"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-a")) {
t.Errorf("expected val-a, got nil")
}
// verify that an iterator sees it
count := 0
it := reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
// add something after the reader was created
writer, err = s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("b"), []byte("val-b"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// ensure that a newer reader sees it
newReader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := newReader.Close()
if err != nil {
t.Fatal(err)
}
}()
val, err = newReader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-b")) {
t.Errorf("expected val-b, got nil")
}
// ensure that the director iterator sees it
count = 0
it = newReader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 2 {
t.Errorf("expected iterator to see 2, saw %d", count)
}
// but that the isolated reader does not
val, err = reader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %v", val)
}
// and ensure that the iterator on the isolated reader also does not
count = 0
it = reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
}

57
vendor/github.com/blevesearch/bleve/index/store/inmem/writer.go generated vendored
View file

@ -1,57 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package inmem
import (
"github.com/blevesearch/bleve/index/store"
)
type Writer struct {
store *Store
}
func newWriter(store *Store) (*Writer, error) {
store.writer.Lock()
return &Writer{
store: store,
}, nil
}
func (w *Writer) BytesSafeAfterClose() bool {
return false
}
func (w *Writer) Set(key, val []byte) error {
return w.store.setlocked(key, val)
}
func (w *Writer) Delete(key []byte) error {
return w.store.deletelocked(key)
}
func (w *Writer) NewBatch() store.KVBatch {
return store.NewEmulatedBatch(w, w.store.mo)
}
func (w *Writer) Close() error {
w.store.writer.Unlock()
return nil
}
// these two methods can safely read using the regular
// methods without a read transaction, because we know
// that no one else is writing but us
func (w *Writer) Get(key []byte) ([]byte, error) {
return w.store.get(key)
}
func (w *Writer) Iterator(key []byte) store.KVIterator {
return w.store.iterator(key)
}

185
vendor/github.com/blevesearch/bleve/index/store/kvstore.go generated vendored
View file

@ -1,53 +1,174 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package store
type KVBatch interface {
Set(key, val []byte)
Delete(key []byte)
Merge(key, val []byte)
Execute() error
import "encoding/json"
// KVStore is an abstraction for working with KV stores. Note that
// in order to be used with the bleve.registry, it must also implement
// a constructor function of the registry.KVStoreConstructor type.
type KVStore interface {
// Writer returns a KVWriter which can be used to
// make changes to the KVStore. If a writer cannot
// be obtained a non-nil error is returned.
Writer() (KVWriter, error)
// Reader returns a KVReader which can be used to
// read data from the KVStore. If a reader cannot
// be obtained a non-nil error is returned.
Reader() (KVReader, error)
// Close closes the KVStore
Close() error
}
// KVReader is an abstraction of an **ISOLATED** reader
// In this context isolated is defined to mean that
// writes/deletes made after the KVReader is opened
// are not observed.
// Because there is usually a cost associated with
// keeping isolated readers active, users should
// close them as soon as they are no longer needed.
type KVReader interface {
// Get returns the value associated with the key
// If the key does not exist, nil is returned.
// The caller owns the bytes returned.
Get(key []byte) ([]byte, error)
// MultiGet retrieves multiple values in one call.
MultiGet(keys [][]byte) ([][]byte, error)
// PrefixIterator returns a KVIterator that will
// visit all K/V pairs with the provided prefix
PrefixIterator(prefix []byte) KVIterator
// RangeIterator returns a KVIterator that will
// visit all K/V pairs >= start AND < end
RangeIterator(start, end []byte) KVIterator
// Close closes the iterator
Close() error
}
// KVIterator is an abstraction around key iteration
type KVIterator interface {
SeekFirst()
Seek([]byte)
// Seek will advance the iterator to the specified key
Seek(key []byte)
// Next will advance the iterator to the next key
Next()
Current() ([]byte, []byte, bool)
// Key returns the key pointed to by the iterator
// The bytes returned are **ONLY** valid until the next call to Seek/Next/Close
// Continued use after that requires that they be copied.
Key() []byte
// Value returns the value pointed to by the iterator
// The bytes returned are **ONLY** valid until the next call to Seek/Next/Close
// Continued use after that requires that they be copied.
Value() []byte
// Valid returns whether or not the iterator is in a valid state
Valid() bool
// Current returns Key(),Value(),Valid() in a single operation
Current() ([]byte, []byte, bool)
// Close closes the iterator
Close() error
}
type KVStore interface {
Open() error
SetMergeOperator(MergeOperator)
Writer() (KVWriter, error)
Reader() (KVReader, error)
Close() error
}
// KVWriter is an abstraction for mutating the KVStore
// KVWriter does **NOT** enforce restrictions of a single writer
// if the underlying KVStore allows concurrent writes, the
// KVWriter interface should also do so, it is up to the caller
// to do this in a way that is safe and makes sense
type KVWriter interface {
KVReader
Set(key, val []byte) error
Delete(key []byte) error
NewBatch() KVBatch
}
type KVReader interface {
BytesSafeAfterClose() bool
Get(key []byte) ([]byte, error)
Iterator(key []byte) KVIterator
// NewBatch returns a KVBatch for performing batch operations on this kvstore
NewBatch() KVBatch
// NewBatchEx returns a KVBatch and an associated byte array
// that's pre-sized based on the KVBatchOptions. The caller can
// use the returned byte array for keys and values associated with
// the batch. Once the batch is either executed or closed, the
// associated byte array should no longer be accessed by the
// caller.
NewBatchEx(KVBatchOptions) ([]byte, KVBatch, error)
// ExecuteBatch will execute the KVBatch, the provided KVBatch **MUST** have
// been created by the same KVStore (though not necessarily the same KVWriter)
// Batch execution is atomic, either all the operations or none will be performed
ExecuteBatch(batch KVBatch) error
// Close closes the writer
Close() error
}
// KVBatchOptions provides the KVWriter.NewBatchEx() method with batch
// preparation and preallocation information.
type KVBatchOptions struct {
// TotalBytes is the sum of key and value bytes needed by the
// caller for the entire batch. It affects the size of the
// returned byte array of KVWrite.NewBatchEx().
TotalBytes int
// NumSets is the number of Set() calls the caller will invoke on
// the KVBatch.
NumSets int
// NumDeletes is the number of Delete() calls the caller will invoke
// on the KVBatch.
NumDeletes int
// NumMerges is the number of Merge() calls the caller will invoke
// on the KVBatch.
NumMerges int
}
// KVBatch is an abstraction for making multiple KV mutations at once
type KVBatch interface {
// Set updates the key with the specified value
// both key and value []byte may be reused as soon as this call returns
Set(key, val []byte)
// Delete removes the specified key
// the key []byte may be reused as soon as this call returns
Delete(key []byte)
// Merge merges old value with the new value at the specified key
// as prescribed by the KVStores merge operator
// both key and value []byte may be reused as soon as this call returns
Merge(key, val []byte)
// Reset frees resources for this batch and allows reuse
Reset()
// Close frees resources
Close() error
}
// KVStoreStats is an optional interface that KVStores can implement
// if they're able to report any useful stats
type KVStoreStats interface {
// Stats returns a JSON serializable object representing stats for this KVStore
Stats() json.Marshaler
StatsMap() map[string]interface{}
}

55
vendor/github.com/blevesearch/bleve/index/store/leveldb/batch.go generated vendored
View file

@ -1,55 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build leveldb full
package leveldb
import (
"github.com/blevesearch/bleve/index/store"
"github.com/jmhodges/levigo"
)
type Batch struct {
w *Writer
merge *store.EmulatedMerge
batch *levigo.WriteBatch
}
func (b *Batch) Set(key, val []byte) {
b.batch.Put(key, val)
}
func (b *Batch) Delete(key []byte) {
b.batch.Delete(key)
}
func (b *Batch) Merge(key, val []byte) {
b.merge.Merge(key, val)
}
func (b *Batch) Execute() error {
// first process merges
ops, err := b.merge.ExecuteDeferred(b.w)
if err != nil {
return err
}
for _, op := range ops {
b.batch.Put(op.K, op.V)
}
wopts := defaultWriteOptions()
defer wopts.Close()
err = b.w.store.db.Write(wopts, b.batch)
return err
}
func (b *Batch) Close() error {
return nil
}

78
vendor/github.com/blevesearch/bleve/index/store/leveldb/iterator.go generated vendored
View file

@ -1,78 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build leveldb full
package leveldb
import (
"github.com/jmhodges/levigo"
)
type Iterator struct {
store *Store
iterator *levigo.Iterator
}
func newIterator(store *Store) *Iterator {
ropts := defaultReadOptions()
rv := Iterator{
store: store,
iterator: store.db.NewIterator(ropts),
}
ropts.Close()
return &rv
}
func newIteratorWithSnapshot(store *Store, snapshot *levigo.Snapshot) *Iterator {
options := defaultReadOptions()
options.SetSnapshot(snapshot)
rv := Iterator{
store: store,
iterator: store.db.NewIterator(options),
}
options.Close()
return &rv
}
func (ldi *Iterator) SeekFirst() {
ldi.iterator.SeekToFirst()
}
func (ldi *Iterator) Seek(key []byte) {
ldi.iterator.Seek(key)
}
func (ldi *Iterator) Next() {
ldi.iterator.Next()
}
func (ldi *Iterator) Current() ([]byte, []byte, bool) {
if ldi.Valid() {
return ldi.Key(), ldi.Value(), true
}
return nil, nil, false
}
func (ldi *Iterator) Key() []byte {
return ldi.iterator.Key()
}
func (ldi *Iterator) Value() []byte {
return ldi.iterator.Value()
}
func (ldi *Iterator) Valid() bool {
return ldi.iterator.Valid()
}
func (ldi *Iterator) Close() error {
ldi.iterator.Close()
return nil
}

48
vendor/github.com/blevesearch/bleve/index/store/leveldb/reader.go generated vendored
View file

@ -1,48 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build leveldb full
package leveldb
import (
"github.com/blevesearch/bleve/index/store"
"github.com/jmhodges/levigo"
)
type Reader struct {
store *Store
snapshot *levigo.Snapshot
}
func newReader(store *Store) (*Reader, error) {
return &Reader{
store: store,
snapshot: store.db.NewSnapshot(),
}, nil
}
func (r *Reader) BytesSafeAfterClose() bool {
return true
}
func (r *Reader) Get(key []byte) ([]byte, error) {
return r.store.getWithSnapshot(key, r.snapshot)
}
func (r *Reader) Iterator(key []byte) store.KVIterator {
rv := newIteratorWithSnapshot(r.store, r.snapshot)
rv.Seek(key)
return rv
}
func (r *Reader) Close() error {
r.store.db.ReleaseSnapshot(r.snapshot)
return nil
}

174
vendor/github.com/blevesearch/bleve/index/store/leveldb/store.go generated vendored
View file

@ -1,174 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build leveldb full
package leveldb
import (
"fmt"
"sync"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/jmhodges/levigo"
)
const Name = "leveldb"
type Store struct {
path string
opts *levigo.Options
db *levigo.DB
writer sync.Mutex
mo store.MergeOperator
}
func New(path string, config map[string]interface{}) (*Store, error) {
rv := Store{
path: path,
opts: levigo.NewOptions(),
}
_, err := applyConfig(rv.opts, config)
if err != nil {
return nil, err
}
return &rv, nil
}
func (ldbs *Store) Open() error {
var err error
ldbs.db, err = levigo.Open(ldbs.path, ldbs.opts)
if err != nil {
return err
}
return nil
}
func (ldbs *Store) SetMergeOperator(mo store.MergeOperator) {
ldbs.mo = mo
}
func (ldbs *Store) get(key []byte) ([]byte, error) {
options := defaultReadOptions()
b, err := ldbs.db.Get(options, key)
options.Close()
return b, err
}
func (ldbs *Store) getWithSnapshot(key []byte, snapshot *levigo.Snapshot) ([]byte, error) {
options := defaultReadOptions()
options.SetSnapshot(snapshot)
b, err := ldbs.db.Get(options, key)
options.Close()
return b, err
}
func (ldbs *Store) set(key, val []byte) error {
ldbs.writer.Lock()
defer ldbs.writer.Unlock()
return ldbs.setlocked(key, val)
}
func (ldbs *Store) setlocked(key, val []byte) error {
options := defaultWriteOptions()
err := ldbs.db.Put(options, key, val)
options.Close()
return err
}
func (ldbs *Store) delete(key []byte) error {
ldbs.writer.Lock()
defer ldbs.writer.Unlock()
return ldbs.deletelocked(key)
}
func (ldbs *Store) deletelocked(key []byte) error {
options := defaultWriteOptions()
err := ldbs.db.Delete(options, key)
options.Close()
return err
}
func (ldbs *Store) Close() error {
ldbs.db.Close()
ldbs.opts.Close()
return nil
}
func (ldbs *Store) iterator(key []byte) store.KVIterator {
rv := newIterator(ldbs)
rv.Seek(key)
return rv
}
func (ldbs *Store) Reader() (store.KVReader, error) {
return newReader(ldbs)
}
func (ldbs *Store) Writer() (store.KVWriter, error) {
return newWriter(ldbs)
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
}
return New(path, config)
}
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
}
func applyConfig(o *levigo.Options, config map[string]interface{}) (
*levigo.Options, error) {
cim, ok := config["create_if_missing"].(bool)
if ok {
o.SetCreateIfMissing(cim)
}
eie, ok := config["error_if_exists"].(bool)
if ok {
o.SetErrorIfExists(eie)
}
wbs, ok := config["write_buffer_size"].(float64)
if ok {
o.SetWriteBufferSize(int(wbs))
}
bs, ok := config["block_size"].(float64)
if ok {
o.SetBlockSize(int(bs))
}
bri, ok := config["block_restart_interval"].(float64)
if ok {
o.SetBlockRestartInterval(int(bri))
}
lcc, ok := config["lru_cache_capacity"].(float64)
if ok {
lruCache := levigo.NewLRUCache(int(lcc))
o.SetCache(lruCache)
}
bfbpk, ok := config["bloom_filter_bits_per_key"].(float64)
if ok {
bf := levigo.NewBloomFilter(int(bfbpk))
o.SetFilterPolicy(bf)
}
return o, nil
}

298
vendor/github.com/blevesearch/bleve/index/store/leveldb/store_test.go generated vendored
View file

@ -1,298 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build leveldb full
package leveldb
import (
"os"
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
)
var leveldbTestOptions = map[string]interface{}{
"create_if_missing": true,
}
func TestLevelDBStore(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
s, err := New("test", leveldbTestOptions)
if err != nil {
t.Fatal(err)
}
err = s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
CommonTestKVStore(t, s)
}
func TestReaderIsolation(t *testing.T) {
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
s, err := New("test", leveldbTestOptions)
if err != nil {
t.Fatal(err)
}
err = s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := s.Close()
if err != nil {
t.Fatal(err)
}
}()
CommonTestReaderIsolation(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
}
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
// insert a kv pair
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// create an isolated reader
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// verify that we see the value already inserted
val, err := reader.Get([]byte("a"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-a")) {
t.Errorf("expected val-a, got nil")
}
// verify that an iterator sees it
count := 0
it := reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
// add something after the reader was created
writer, err = s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("b"), []byte("val-b"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// ensure that a newer reader sees it
newReader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := newReader.Close()
if err != nil {
t.Fatal(err)
}
}()
val, err = newReader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-b")) {
t.Errorf("expected val-b, got nil")
}
// ensure that the director iterator sees it
count = 0
it2 := newReader.Iterator([]byte{0})
defer func() {
err := it2.Close()
if err != nil {
t.Fatal(err)
}
}()
for it2.Valid() {
it2.Next()
count++
}
if count != 2 {
t.Errorf("expected iterator to see 2, saw %d", count)
}
// but that the isolated reader does not
val, err = reader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %v", val)
}
// and ensure that the iterator on the isolated reader also does not
count = 0
it3 := reader.Iterator([]byte{0})
defer func() {
err := it3.Close()
if err != nil {
t.Fatal(err)
}
}()
for it3.Valid() {
it3.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
}

28
vendor/github.com/blevesearch/bleve/index/store/leveldb/util.go generated vendored
View file

@ -1,28 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build leveldb full
package leveldb
import (
"github.com/jmhodges/levigo"
)
func defaultWriteOptions() *levigo.WriteOptions {
wo := levigo.NewWriteOptions()
// request fsync on write for safety
wo.SetSync(true)
return wo
}
func defaultReadOptions() *levigo.ReadOptions {
ro := levigo.NewReadOptions()
return ro
}

65
vendor/github.com/blevesearch/bleve/index/store/leveldb/writer.go generated vendored
View file

@ -1,65 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
// +build leveldb full
package leveldb
import (
"github.com/blevesearch/bleve/index/store"
"github.com/jmhodges/levigo"
)
type Writer struct {
store *Store
}
func newWriter(store *Store) (*Writer, error) {
store.writer.Lock()
return &Writer{
store: store,
}, nil
}
func (w *Writer) BytesSafeAfterClose() bool {
return true
}
func (w *Writer) Set(key, val []byte) error {
return w.store.setlocked(key, val)
}
func (w *Writer) Delete(key []byte) error {
return w.store.deletelocked(key)
}
func (w *Writer) NewBatch() store.KVBatch {
rv := Batch{
w: w,
merge: store.NewEmulatedMerge(w.store.mo),
batch: levigo.NewWriteBatch(),
}
return &rv
}
func (w *Writer) Close() error {
w.store.writer.Unlock()
return nil
}
// these two methods can safely read using the regular
// methods without a read transaction, because we know
// that no one else is writing but us
func (w *Writer) Get(key []byte) ([]byte, error) {
return w.store.get(key)
}
func (w *Writer) Iterator(key []byte) store.KVIterator {
return w.store.iterator(key)
}

88
vendor/github.com/blevesearch/bleve/index/store/merge.go generated vendored
View file

@ -1,18 +1,19 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package store
import (
"fmt"
)
// At the moment this happens to be the same interface as described by
// RocksDB, but this may not always be the case.
@ -32,41 +33,20 @@ type MergeOperator interface {
Name() string
}
// EmulatedMergeSingle removes some duplicated code across
// KV stores which do not support merge operations
// on their own. It is up to the caller to ensure
// that an appropriate lock has been acquired in
// order for this behavior to be valid
func EmulatedMergeSingle(writer KVWriter, mo MergeOperator, key []byte, operand []byte) error {
existingValue, err := writer.Get(key)
if err != nil {
return err
}
newValue, ok := mo.FullMerge(key, existingValue, [][]byte{operand})
if !ok {
return fmt.Errorf("merge operator returned failure")
}
err = writer.Set(key, newValue)
if err != nil {
return err
}
return nil
}
type EmulatedMerge struct {
merges map[string][][]byte
Merges map[string][][]byte
mo MergeOperator
}
func NewEmulatedMerge(mo MergeOperator) *EmulatedMerge {
return &EmulatedMerge{
merges: make(map[string][][]byte),
Merges: make(map[string][][]byte),
mo: mo,
}
}
func (m *EmulatedMerge) Merge(key, val []byte) {
ops, ok := m.merges[string(key)]
ops, ok := m.Merges[string(key)]
if ok && len(ops) > 0 {
last := ops[len(ops)-1]
mergedVal, partialMergeOk := m.mo.PartialMerge(key, last, val)
@ -80,41 +60,5 @@ func (m *EmulatedMerge) Merge(key, val []byte) {
} else {
ops = [][]byte{val}
}
m.merges[string(key)] = ops
}
func (m *EmulatedMerge) Execute(w KVWriter) error {
for k, mergeOps := range m.merges {
kb := []byte(k)
existingVal, err := w.Get(kb)
if err != nil {
return err
}
mergedVal, fullMergeOk := m.mo.FullMerge(kb, existingVal, mergeOps)
if !fullMergeOk {
return fmt.Errorf("merge operator returned failure")
}
err = w.Set(kb, mergedVal)
if err != nil {
return err
}
}
return nil
}
func (m *EmulatedMerge) ExecuteDeferred(w KVWriter) ([]*op, error) {
rv := make([]*op, 0, 1000)
for k, mergeOps := range m.merges {
kb := []byte(k)
existingVal, err := w.Get(kb)
if err != nil {
return nil, err
}
mergedVal, fullMergeOk := m.mo.FullMerge(kb, existingVal, mergeOps)
if !fullMergeOk {
return nil, fmt.Errorf("merge operator returned failure")
}
rv = append(rv, &op{kb, mergedVal})
}
return rv, nil
m.Merges[string(key)] = ops
}

476
vendor/github.com/blevesearch/bleve/index/store/metrics/metrics.go generated vendored
View file

@ -1,476 +0,0 @@
// Copyright (c) 2015 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
// +build debug
// Package metrics provides a bleve.store.KVStore implementation that
// wraps another, real KVStore implementation, and uses go-metrics to
// track runtime performance metrics.
package metrics
import (
"container/list"
"encoding/json"
"fmt"
"io"
"sync"
"time"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/rcrowley/go-metrics"
)
const Name = "metrics"
const MaxErrors = 100
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
name, ok := config["kvStoreName_actual"].(string)
if !ok || name == "" {
return nil, fmt.Errorf("metrics: missing kvStoreName_actual,"+
" config: %#v", config)
}
if name == Name {
return nil, fmt.Errorf("metrics: circular kvStoreName_actual")
}
ctr := registry.KVStoreConstructorByName(name)
if ctr == nil {
return nil, fmt.Errorf("metrics: no kv store constructor,"+
" kvStoreName_actual: %s", name)
}
kvs, err := ctr(config)
if err != nil {
return nil, err
}
return NewBleveMetricsStore(kvs), nil
}
func NewBleveMetricsStore(o store.KVStore) *Store {
return &Store{
o: o,
TimerReaderGet: metrics.NewTimer(),
TimerReaderIterator: metrics.NewTimer(),
TimerWriterGet: metrics.NewTimer(),
TimerWriterIterator: metrics.NewTimer(),
TimerWriterSet: metrics.NewTimer(),
TimerWriterDelete: metrics.NewTimer(),
TimerIteratorSeekFirst: metrics.NewTimer(),
TimerIteratorSeek: metrics.NewTimer(),
TimerIteratorNext: metrics.NewTimer(),
TimerBatchMerge: metrics.NewTimer(),
TimerBatchExecute: metrics.NewTimer(),
errors: list.New(),
}
}
// The following structs are wrappers around "real" bleve kvstore
// implementations.
type Store struct {
o store.KVStore
TimerReaderGet metrics.Timer
TimerReaderIterator metrics.Timer
TimerWriterGet metrics.Timer
TimerWriterIterator metrics.Timer
TimerWriterSet metrics.Timer
TimerWriterDelete metrics.Timer
TimerIteratorSeekFirst metrics.Timer
TimerIteratorSeek metrics.Timer
TimerIteratorNext metrics.Timer
TimerBatchMerge metrics.Timer
TimerBatchExecute metrics.Timer
m sync.Mutex // Protects the fields that follow.
errors *list.List // Capped list of StoreError's.
}
type StoreError struct {
Time string
Op string
Err string
Key string
}
type Reader struct {
s *Store
o store.KVReader
}
type Writer struct {
s *Store
o store.KVWriter
}
type Iterator struct {
s *Store
o store.KVIterator
}
type Batch struct {
s *Store
o store.KVBatch
}
func (s *Store) Close() error {
return s.o.Close()
}
func (s *Store) Reader() (store.KVReader, error) {
o, err := s.o.Reader()
if err != nil {
s.AddError("Reader", err, nil)
return nil, err
}
return &Reader{s: s, o: o}, nil
}
func (s *Store) Writer() (store.KVWriter, error) {
o, err := s.o.Writer()
if err != nil {
s.AddError("Writer", err, nil)
return nil, err
}
return &Writer{s: s, o: o}, nil
}
func (s *Store) Actual() store.KVStore {
return s.o
}
func (w *Reader) BytesSafeAfterClose() bool {
return w.o.BytesSafeAfterClose()
}
func (w *Reader) Get(key []byte) (v []byte, err error) {
w.s.TimerReaderGet.Time(func() {
v, err = w.o.Get(key)
if err != nil {
w.s.AddError("Reader.Get", err, key)
}
})
return
}
func (w *Reader) Iterator(key []byte) (i store.KVIterator) {
w.s.TimerReaderIterator.Time(func() {
i = &Iterator{s: w.s, o: w.o.Iterator(key)}
})
return
}
func (w *Reader) Close() error {
err := w.o.Close()
if err != nil {
w.s.AddError("Reader.Close", err, nil)
}
return err
}
func (w *Writer) BytesSafeAfterClose() bool {
return w.o.BytesSafeAfterClose()
}
func (w *Writer) Get(key []byte) (v []byte, err error) {
w.s.TimerWriterGet.Time(func() {
v, err = w.o.Get(key)
if err != nil {
w.s.AddError("Writer.Get", err, key)
}
})
return
}
func (w *Writer) Iterator(key []byte) (i store.KVIterator) {
w.s.TimerWriterIterator.Time(func() {
i = &Iterator{s: w.s, o: w.o.Iterator(key)}
})
return
}
func (w *Writer) Close() error {
err := w.o.Close()
if err != nil {
w.s.AddError("Writer.Close", err, nil)
}
return err
}
func (w *Writer) Set(key, val []byte) (err error) {
w.s.TimerWriterSet.Time(func() {
err = w.o.Set(key, val)
if err != nil {
w.s.AddError("Writer.Set", err, key)
}
})
return
}
func (w *Writer) Delete(key []byte) (err error) {
w.s.TimerWriterDelete.Time(func() {
err = w.o.Delete(key)
if err != nil {
w.s.AddError("Writer.Delete", err, key)
}
})
return
}
func (w *Writer) NewBatch() store.KVBatch {
return &Batch{s: w.s, o: w.o.NewBatch()}
}
func (w *Iterator) SeekFirst() {
w.s.TimerIteratorSeekFirst.Time(func() {
w.o.SeekFirst()
})
}
func (w *Iterator) Seek(x []byte) {
w.s.TimerIteratorSeek.Time(func() {
w.o.Seek(x)
})
}
func (w *Iterator) Next() {
w.s.TimerIteratorNext.Time(func() {
w.o.Next()
})
}
func (w *Iterator) Current() ([]byte, []byte, bool) {
return w.o.Current()
}
func (w *Iterator) Key() []byte {
return w.o.Key()
}
func (w *Iterator) Value() []byte {
return w.o.Value()
}
func (w *Iterator) Valid() bool {
return w.o.Valid()
}
func (w *Iterator) Close() error {
err := w.o.Close()
if err != nil {
w.s.AddError("Iterator.Close", err, nil)
}
return err
}
func (w *Batch) Set(key, val []byte) {
w.o.Set(key, val)
}
func (w *Batch) Delete(key []byte) {
w.o.Delete(key)
}
func (w *Batch) Merge(key []byte, oper store.AssociativeMerge) {
w.s.TimerBatchMerge.Time(func() {
w.o.Merge(key, oper)
})
}
func (w *Batch) Execute() (err error) {
w.s.TimerBatchExecute.Time(func() {
err = w.o.Execute()
if err != nil {
w.s.AddError("Batch.Execute", err, nil)
}
})
return
}
func (w *Batch) Close() error {
err := w.o.Close()
if err != nil {
w.s.AddError("Batch.Close", err, nil)
}
return err
}
// --------------------------------------------------------
func (s *Store) AddError(op string, err error, key []byte) {
e := &StoreError{
Time: time.Now().Format(time.RFC3339Nano),
Op: op,
Err: fmt.Sprintf("%v", err),
Key: string(key),
}
s.m.Lock()
for s.errors.Len() >= MaxErrors {
s.errors.Remove(s.errors.Front())
}
s.errors.PushBack(e)
s.m.Unlock()
}
// --------------------------------------------------------
func (s *Store) WriteJSON(w io.Writer) {
w.Write([]byte(`{"TimerReaderGet":`))
WriteTimerJSON(w, s.TimerReaderGet)
w.Write([]byte(`,"TimerReaderIterator":`))
WriteTimerJSON(w, s.TimerReaderIterator)
w.Write([]byte(`,"TimerWriterGet":`))
WriteTimerJSON(w, s.TimerWriterGet)
w.Write([]byte(`,"TimerWriterIterator":`))
WriteTimerJSON(w, s.TimerWriterIterator)
w.Write([]byte(`,"TimerWriterSet":`))
WriteTimerJSON(w, s.TimerWriterSet)
w.Write([]byte(`,"TimerWriterDelete":`))
WriteTimerJSON(w, s.TimerWriterDelete)
w.Write([]byte(`,"TimerIteratorSeekFirst":`))
WriteTimerJSON(w, s.TimerIteratorSeekFirst)
w.Write([]byte(`,"TimerIteratorSeek":`))
WriteTimerJSON(w, s.TimerIteratorSeek)
w.Write([]byte(`,"TimerIteratorNext":`))
WriteTimerJSON(w, s.TimerIteratorNext)
w.Write([]byte(`,"TimerBatchMerge":`))
WriteTimerJSON(w, s.TimerBatchMerge)
w.Write([]byte(`,"TimerBatchExecute":`))
WriteTimerJSON(w, s.TimerBatchExecute)
w.Write([]byte(`,"Errors":[`))
s.m.Lock()
e := s.errors.Front()
i := 0
for e != nil {
se, ok := e.Value.(*StoreError)
if ok && se != nil {
if i > 0 {
w.Write([]byte(","))
}
buf, err := json.Marshal(se)
if err == nil {
w.Write(buf)
}
}
e = e.Next()
i = i + 1
}
s.m.Unlock()
w.Write([]byte(`]`))
w.Write([]byte(`}`))
}
func (s *Store) WriteCSVHeader(w io.Writer) {
WriteTimerCSVHeader(w, "TimerReaderGet")
WriteTimerCSVHeader(w, "TimerReaderIterator")
WriteTimerCSVHeader(w, "TimerWriterGet")
WriteTimerCSVHeader(w, "TimerWriterIterator")
WriteTimerCSVHeader(w, "TimerWriterSet")
WriteTimerCSVHeader(w, "TimerWriterDelete")
WriteTimerCSVHeader(w, "TimerIteratorSeekFirst")
WriteTimerCSVHeader(w, "TimerIteratorSeek")
WriteTimerCSVHeader(w, "TimerIteratorNext")
WriteTimerCSVHeader(w, "TimerBatchMerge")
WriteTimerCSVHeader(w, "TimerBatchExecute")
}
func (s *Store) WriteCSV(w io.Writer) {
WriteTimerCSV(w, s.TimerReaderGet)
WriteTimerCSV(w, s.TimerReaderIterator)
WriteTimerCSV(w, s.TimerWriterGet)
WriteTimerCSV(w, s.TimerWriterIterator)
WriteTimerCSV(w, s.TimerWriterSet)
WriteTimerCSV(w, s.TimerWriterDelete)
WriteTimerCSV(w, s.TimerIteratorSeekFirst)
WriteTimerCSV(w, s.TimerIteratorSeek)
WriteTimerCSV(w, s.TimerIteratorNext)
WriteTimerCSV(w, s.TimerBatchMerge)
WriteTimerCSV(w, s.TimerBatchExecute)
}
// --------------------------------------------------------
// NOTE: This is copy & pasted from cbft as otherwise there
// would be an import cycle.
var timerPercentiles = []float64{0.5, 0.75, 0.95, 0.99, 0.999}
func WriteTimerJSON(w io.Writer, timer metrics.Timer) {
t := timer.Snapshot()
p := t.Percentiles(timerPercentiles)
fmt.Fprintf(w, `{"count":%9d,`, t.Count())
fmt.Fprintf(w, `"min":%9d,`, t.Min())
fmt.Fprintf(w, `"max":%9d,`, t.Max())
fmt.Fprintf(w, `"mean":%12.2f,`, t.Mean())
fmt.Fprintf(w, `"stddev":%12.2f,`, t.StdDev())
fmt.Fprintf(w, `"percentiles":{`)
fmt.Fprintf(w, `"median":%12.2f,`, p[0])
fmt.Fprintf(w, `"75%%":%12.2f,`, p[1])
fmt.Fprintf(w, `"95%%":%12.2f,`, p[2])
fmt.Fprintf(w, `"99%%":%12.2f,`, p[3])
fmt.Fprintf(w, `"99.9%%":%12.2f},`, p[4])
fmt.Fprintf(w, `"rates":{`)
fmt.Fprintf(w, `"1-min":%12.2f,`, t.Rate1())
fmt.Fprintf(w, `"5-min":%12.2f,`, t.Rate5())
fmt.Fprintf(w, `"15-min":%12.2f,`, t.Rate15())
fmt.Fprintf(w, `"mean":%12.2f}}`, t.RateMean())
}
func WriteTimerCSVHeader(w io.Writer, prefix string) {
fmt.Fprintf(w, "%s-count,", prefix)
fmt.Fprintf(w, "%s-min,", prefix)
fmt.Fprintf(w, "%s-max,", prefix)
fmt.Fprintf(w, "%s-mean,", prefix)
fmt.Fprintf(w, "%s-stddev,", prefix)
fmt.Fprintf(w, "%s-percentile-50%%,", prefix)
fmt.Fprintf(w, "%s-percentile-75%%,", prefix)
fmt.Fprintf(w, "%s-percentile-95%%,", prefix)
fmt.Fprintf(w, "%s-percentile-99%%,", prefix)
fmt.Fprintf(w, "%s-percentile-99.9%%,", prefix)
fmt.Fprintf(w, "%s-rate-1-min,", prefix)
fmt.Fprintf(w, "%s-rate-5-min,", prefix)
fmt.Fprintf(w, "%s-rate-15-min,", prefix)
fmt.Fprintf(w, "%s-rate-mean", prefix)
}
func WriteTimerCSV(w io.Writer, timer metrics.Timer) {
t := timer.Snapshot()
p := t.Percentiles(timerPercentiles)
fmt.Fprintf(w, `%d,`, t.Count())
fmt.Fprintf(w, `%d,`, t.Min())
fmt.Fprintf(w, `%d,`, t.Max())
fmt.Fprintf(w, `%f,`, t.Mean())
fmt.Fprintf(w, `%f,`, t.StdDev())
fmt.Fprintf(w, `%f,`, p[0])
fmt.Fprintf(w, `%f,`, p[1])
fmt.Fprintf(w, `%f,`, p[2])
fmt.Fprintf(w, `%f,`, p[3])
fmt.Fprintf(w, `%f,`, p[4])
fmt.Fprintf(w, `%f,`, t.Rate1())
fmt.Fprintf(w, `%f,`, t.Rate5())
fmt.Fprintf(w, `%f,`, t.Rate15())
fmt.Fprintf(w, `%f`, t.RateMean())
}

368
vendor/github.com/blevesearch/bleve/index/store/metrics/metrics_test.go generated vendored
View file

@ -1,368 +0,0 @@
// Copyright (c) 2015 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the
// License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an "AS
// IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language
// governing permissions and limitations under the License.
// +build debug
package metrics
import (
"bytes"
"encoding/json"
"fmt"
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
_ "github.com/blevesearch/bleve/index/store/gtreap"
)
func TestMetricsStore(t *testing.T) {
s, err := StoreConstructor(map[string]interface{}{})
if err == nil {
t.Errorf("expected err when bad config")
}
s, err = StoreConstructor(map[string]interface{}{
"kvStoreName_actual": "some-invalid-kvstore-name",
})
if err == nil {
t.Errorf("expected err when unknown kvStoreName_actual")
}
s, err = StoreConstructor(map[string]interface{}{
"kvStoreName_actual": "gtreap",
})
if err != nil {
t.Fatal(err)
}
CommonTestKVStore(t, s)
b := bytes.NewBuffer(nil)
s.(*Store).WriteJSON(b)
if b.Len() <= 0 {
t.Errorf("expected some output from WriteJSON")
}
var m map[string]interface{}
err = json.Unmarshal(b.Bytes(), &m)
if err != nil {
t.Errorf("expected WriteJSON to be unmarshallable")
}
if len(m) <= 0 {
t.Errorf("expected some entries")
}
b = bytes.NewBuffer(nil)
s.(*Store).WriteCSVHeader(b)
if b.Len() <= 0 {
t.Errorf("expected some output from WriteCSVHeader")
}
b = bytes.NewBuffer(nil)
s.(*Store).WriteCSV(b)
if b.Len() <= 0 {
t.Errorf("expected some output from WriteCSV")
}
}
func TestReaderIsolation(t *testing.T) {
s, err := StoreConstructor(map[string]interface{}{
"kvStoreName_actual": "gtreap",
})
if err != nil {
t.Fatal(err)
}
CommonTestReaderIsolation(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
}
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
// insert a kv pair
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// create an isolated reader
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// verify that we see the value already inserted
val, err := reader.Get([]byte("a"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-a")) {
t.Errorf("expected val-a, got nil")
}
// verify that an iterator sees it
count := 0
it := reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
// add something after the reader was created
writer, err = s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("b"), []byte("val-b"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// ensure that a newer reader sees it
newReader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := newReader.Close()
if err != nil {
t.Fatal(err)
}
}()
val, err = newReader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-b")) {
t.Errorf("expected val-b, got nil")
}
// ensure that the director iterator sees it
count = 0
it2 := newReader.Iterator([]byte{0})
defer func() {
err := it2.Close()
if err != nil {
t.Fatal(err)
}
}()
for it2.Valid() {
it2.Next()
count++
}
if count != 2 {
t.Errorf("expected iterator to see 2, saw %d", count)
}
// but that the isolated reader does not
val, err = reader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %v", val)
}
// and ensure that the iterator on the isolated reader also does not
count = 0
it3 := reader.Iterator([]byte{0})
defer func() {
err := it3.Close()
if err != nil {
t.Fatal(err)
}
}()
for it3.Valid() {
it3.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
}
func TestErrors(t *testing.T) {
s, err := StoreConstructor(map[string]interface{}{
"kvStoreName_actual": "gtreap",
})
if err != nil {
t.Fatal(err)
}
x, ok := s.(*Store)
if !ok {
t.Errorf("expecting a Store")
}
x.AddError("foo", fmt.Errorf("Foo"), []byte("fooKey"))
x.AddError("bar", fmt.Errorf("Bar"), nil)
x.AddError("baz", fmt.Errorf("Baz"), []byte("bazKey"))
b := bytes.NewBuffer(nil)
x.WriteJSON(b)
var m map[string]interface{}
err = json.Unmarshal(b.Bytes(), &m)
if err != nil {
t.Errorf("expected unmarshallable writeJSON, err: %v, b: %s",
err, b.Bytes())
}
errorsi, ok := m["Errors"]
if !ok || errorsi == nil {
t.Errorf("expected errorsi")
}
errors, ok := errorsi.([]interface{})
if !ok || errors == nil {
t.Errorf("expected errorsi is array")
}
if len(errors) != 3 {
t.Errorf("expected errors len 3")
}
e := errors[0].(map[string]interface{})
if e["Op"].(string) != "foo" ||
e["Err"].(string) != "Foo" ||
len(e["Time"].(string)) < 10 ||
e["Key"].(string) != "fooKey" {
t.Errorf("expected foo, %#v", e)
}
e = errors[1].(map[string]interface{})
if e["Op"].(string) != "bar" ||
e["Err"].(string) != "Bar" ||
len(e["Time"].(string)) < 10 ||
e["Key"].(string) != "" {
t.Errorf("expected bar, %#v", e)
}
e = errors[2].(map[string]interface{})
if e["Op"].(string) != "baz" ||
e["Err"].(string) != "Baz" ||
len(e["Time"].(string)) < 10 ||
e["Key"].(string) != "bazKey" {
t.Errorf("expected baz, %#v", e)
}
}

33
vendor/github.com/blevesearch/bleve/index/store/multiget.go generated vendored Normal file
View file

@ -0,0 +1,33 @@
// Copyright (c) 2016 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package store
// MultiGet is a helper function to retrieve mutiple keys from a
// KVReader, and might be used by KVStore implementations that don't
// have a native multi-get facility.
func MultiGet(kvreader KVReader, keys [][]byte) ([][]byte, error) {
vals := make([][]byte, 0, len(keys))
for i, key := range keys {
val, err := kvreader.Get(key)
if err != nil {
return nil, err
}
vals[i] = val
}
return vals, nil
}

173
vendor/github.com/blevesearch/bleve/index/store/null/null.go generated vendored
View file

@ -1,173 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package null
import (
"github.com/blevesearch/bleve/index/store"
)
type Store struct{}
func New() (*Store, error) {
rv := Store{}
return &rv, nil
}
func (i *Store) Open() error {
return nil
}
func (i *Store) SetMergeOperator(mo store.MergeOperator) {
}
func (i *Store) Close() error {
return nil
}
func (i *Store) iterator(key []byte) store.KVIterator {
rv := newIterator(i)
return rv
}
func (i *Store) Reader() (store.KVReader, error) {
return newReader(i)
}
func (i *Store) Writer() (store.KVWriter, error) {
return newWriter(i)
}
func (i *Store) newBatch() store.KVBatch {
return newBatch(i)
}
type Reader struct {
store *Store
}
func newReader(store *Store) (*Reader, error) {
return &Reader{
store: store,
}, nil
}
func (r *Reader) BytesSafeAfterClose() bool {
return true
}
func (r *Reader) Get(key []byte) ([]byte, error) {
return nil, nil
}
func (r *Reader) Iterator(key []byte) store.KVIterator {
return r.store.iterator(key)
}
func (r *Reader) Close() error {
return nil
}
type Iterator struct{}
func newIterator(store *Store) *Iterator {
return &Iterator{}
}
func (i *Iterator) SeekFirst() {}
func (i *Iterator) Seek(k []byte) {}
func (i *Iterator) Next() {}
func (i *Iterator) Current() ([]byte, []byte, bool) {
return nil, nil, false
}
func (i *Iterator) Key() []byte {
return nil
}
func (i *Iterator) Value() []byte {
return nil
}
func (i *Iterator) Valid() bool {
return false
}
func (i *Iterator) Close() error {
return nil
}
type Batch struct{}
func newBatch(s *Store) *Batch {
rv := Batch{}
return &rv
}
func (i *Batch) Set(key, val []byte) {
}
func (i *Batch) Delete(key []byte) {
}
func (i *Batch) Merge(key, val []byte) {
}
func (i *Batch) Execute() error {
return nil
}
func (i *Batch) Close() error {
return nil
}
type Writer struct {
store *Store
}
func newWriter(store *Store) (*Writer, error) {
return &Writer{
store: store,
}, nil
}
func (w *Writer) BytesSafeAfterClose() bool {
return true
}
func (w *Writer) Set(key, val []byte) error {
return nil
}
func (w *Writer) Delete(key []byte) error {
return nil
}
func (w *Writer) NewBatch() store.KVBatch {
return newBatch(w.store)
}
func (w *Writer) Close() error {
return nil
}
// these two methods can safely read using the regular
// methods without a read transaction, because we know
// that no one else is writing but us
func (w *Writer) Get(key []byte) ([]byte, error) {
return nil, nil
}
func (w *Writer) Iterator(key []byte) store.KVIterator {
return w.store.iterator(key)
}

88
vendor/github.com/blevesearch/bleve/index/store/null/null_test.go generated vendored
View file

@ -1,88 +0,0 @@
package null
import (
"testing"
"github.com/blevesearch/bleve/index/store"
)
func TestStore(t *testing.T) {
s, err := New()
if err != nil {
t.Fatal(err)
}
CommonTestKVStore(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if valid {
t.Fatalf("valid true, expected false")
}
if key != nil {
t.Fatalf("expected key nil, got %s", key)
}
if val != nil {
t.Fatalf("expected value nil, got %s", val)
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
err = s.Close()
if err != nil {
t.Fatal(err)
}
}