Upgrade server dependencies, manage them with govendor

This commit is contained in:
Ken-Håvard Lieng 2017-04-18 03:02:51 +02:00
parent ebee2746d6
commit 971278e7e5
1748 changed files with 196165 additions and 194500 deletions

View file

@ -1,15 +1,22 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package boltdb
import (
"bytes"
"github.com/boltdb/bolt"
)
@ -17,24 +24,44 @@ type Iterator struct {
store *Store
tx *bolt.Tx
cursor *bolt.Cursor
prefix []byte
start []byte
end []byte
valid bool
key []byte
val []byte
}
func (i *Iterator) SeekFirst() {
i.key, i.val = i.cursor.First()
func (i *Iterator) updateValid() {
i.valid = (i.key != nil)
if i.valid {
if i.prefix != nil {
i.valid = bytes.HasPrefix(i.key, i.prefix)
} else if i.end != nil {
i.valid = bytes.Compare(i.key, i.end) < 0
}
}
}
func (i *Iterator) Seek(k []byte) {
if i.start != nil && bytes.Compare(k, i.start) < 0 {
k = i.start
}
if i.prefix != nil && !bytes.HasPrefix(k, i.prefix) {
if bytes.Compare(k, i.prefix) < 0 {
k = i.prefix
} else {
i.valid = false
return
}
}
i.key, i.val = i.cursor.Seek(k)
i.valid = (i.key != nil)
i.updateValid()
}
func (i *Iterator) Next() {
i.key, i.val = i.cursor.Next()
i.valid = (i.key != nil)
i.updateValid()
}
func (i *Iterator) Current() ([]byte, []byte, bool) {

View file

@ -1,11 +1,16 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package boltdb
@ -15,30 +20,51 @@ import (
)
type Reader struct {
store *Store
tx *bolt.Tx
}
func (r *Reader) BytesSafeAfterClose() bool {
return false
store *Store
tx *bolt.Tx
bucket *bolt.Bucket
}
func (r *Reader) Get(key []byte) ([]byte, error) {
rv := r.tx.Bucket([]byte(r.store.bucket)).Get(key)
var rv []byte
v := r.bucket.Get(key)
if v != nil {
rv = make([]byte, len(v))
copy(rv, v)
}
return rv, nil
}
func (r *Reader) Iterator(key []byte) store.KVIterator {
b := r.tx.Bucket([]byte(r.store.bucket))
cursor := b.Cursor()
func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {
return store.MultiGet(r, keys)
}
func (r *Reader) PrefixIterator(prefix []byte) store.KVIterator {
cursor := r.bucket.Cursor()
rv := &Iterator{
store: r.store,
tx: r.tx,
cursor: cursor,
prefix: prefix,
}
rv.Seek(key)
rv.Seek(prefix)
return rv
}
func (r *Reader) RangeIterator(start, end []byte) store.KVIterator {
cursor := r.bucket.Cursor()
rv := &Iterator{
store: r.store,
tx: r.tx,
cursor: cursor,
start: start,
end: end,
}
rv.Seek(start)
return rv
}

View file

@ -0,0 +1,26 @@
// Copyright (c) 2014 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package boltdb
import "encoding/json"
type stats struct {
s *Store
}
func (s *stats) MarshalJSON() ([]byte, error) {
bs := s.s.db.Stats()
return json.Marshal(bs)
}

View file

@ -1,63 +1,105 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package boltdb implements a store.KVStore on top of BoltDB. It supports the
// following options:
//
// "bucket" (string): the name of BoltDB bucket to use, defaults to "bleve".
//
// "nosync" (bool): if true, set boltdb.DB.NoSync to true. It speeds up index
// operations in exchange of losing integrity guarantees if indexation aborts
// without closing the index. Use it when rebuilding indexes from zero.
package boltdb
import (
"bytes"
"encoding/json"
"fmt"
"sync"
"os"
"github.com/blevesearch/bleve/index/store"
"github.com/blevesearch/bleve/registry"
"github.com/boltdb/bolt"
)
const Name = "boltdb"
const (
Name = "boltdb"
defaultCompactBatchSize = 100
)
type Store struct {
path string
bucket string
db *bolt.DB
writer sync.Mutex
mo store.MergeOperator
path string
bucket string
db *bolt.DB
noSync bool
fillPercent float64
mo store.MergeOperator
}
func New(path string, bucket string) *Store {
func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
}
if path == "" {
return nil, os.ErrInvalid
}
bucket, ok := config["bucket"].(string)
if !ok {
bucket = "bleve"
}
noSync, _ := config["nosync"].(bool)
fillPercent, ok := config["fillPercent"].(float64)
if !ok {
fillPercent = bolt.DefaultFillPercent
}
bo := &bolt.Options{}
ro, ok := config["read_only"].(bool)
if ok {
bo.ReadOnly = ro
}
db, err := bolt.Open(path, 0600, bo)
if err != nil {
return nil, err
}
db.NoSync = noSync
if !bo.ReadOnly {
err = db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(bucket))
return err
})
if err != nil {
return nil, err
}
}
rv := Store{
path: path,
bucket: bucket,
path: path,
bucket: bucket,
db: db,
mo: mo,
noSync: noSync,
fillPercent: fillPercent,
}
return &rv
}
func (bs *Store) Open() error {
var err error
bs.db, err = bolt.Open(bs.path, 0600, nil)
if err != nil {
return err
}
err = bs.db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucketIfNotExists([]byte(bs.bucket))
return err
})
if err != nil {
return err
}
return nil
}
func (bs *Store) SetMergeOperator(mo store.MergeOperator) {
bs.mo = mo
return &rv, nil
}
func (bs *Store) Close() error {
@ -70,43 +112,64 @@ func (bs *Store) Reader() (store.KVReader, error) {
return nil, err
}
return &Reader{
store: bs,
tx: tx,
store: bs,
tx: tx,
bucket: tx.Bucket([]byte(bs.bucket)),
}, nil
}
func (bs *Store) Writer() (store.KVWriter, error) {
bs.writer.Lock()
tx, err := bs.db.Begin(true)
if err != nil {
bs.writer.Unlock()
return nil, err
}
reader := &Reader{
store: bs,
tx: tx,
}
return &Writer{
store: bs,
tx: tx,
reader: reader,
store: bs,
}, nil
}
func StoreConstructor(config map[string]interface{}) (store.KVStore, error) {
path, ok := config["path"].(string)
if !ok {
return nil, fmt.Errorf("must specify path")
func (bs *Store) Stats() json.Marshaler {
return &stats{
s: bs,
}
}
bucket, ok := config["bucket"].(string)
if !ok {
bucket = "bleve"
// CompactWithBatchSize removes DictionaryTerm entries with a count of zero (in batchSize batches)
// Removing entries is a workaround for github issue #374.
func (bs *Store) CompactWithBatchSize(batchSize int) error {
for {
cnt := 0
err := bs.db.Batch(func(tx *bolt.Tx) error {
c := tx.Bucket([]byte(bs.bucket)).Cursor()
prefix := []byte("d")
for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() {
if bytes.Equal(v, []byte{0}) {
cnt++
if err := c.Delete(); err != nil {
return err
}
if cnt == batchSize {
break
}
}
}
return nil
})
if err != nil {
return err
}
if cnt == 0 {
break
}
}
return nil
}
return New(path, bucket), nil
// Compact calls CompactWithBatchSize with a default batch size of 100. This is a workaround
// for github issue #374.
func (bs *Store) Compact() error {
return bs.CompactWithBatchSize(defaultCompactBatchSize)
}
func init() {
registry.RegisterKVStore(Name, StoreConstructor)
registry.RegisterKVStore(Name, New)
}

View file

@ -1,272 +0,0 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
package boltdb
import (
"os"
"reflect"
"testing"
"github.com/blevesearch/bleve/index/store"
)
func TestStore(t *testing.T) {
s := New("test", "bleve")
err := s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
CommonTestKVStore(t, s)
}
func TestReaderIsolation(t *testing.T) {
s := New("test", "bleve")
err := s.Open()
if err != nil {
t.Fatal(err)
}
defer func() {
err := os.RemoveAll("test")
if err != nil {
t.Fatal(err)
}
}()
CommonTestReaderIsolation(t, s)
}
func CommonTestKVStore(t *testing.T, s store.KVStore) {
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Set([]byte("z"), []byte("val-z"))
if err != nil {
t.Fatal(err)
}
err = writer.Delete([]byte("z"))
if err != nil {
t.Fatal(err)
}
batch := writer.NewBatch()
batch.Set([]byte("b"), []byte("val-b"))
batch.Set([]byte("c"), []byte("val-c"))
batch.Set([]byte("d"), []byte("val-d"))
batch.Set([]byte("e"), []byte("val-e"))
batch.Set([]byte("f"), []byte("val-f"))
batch.Set([]byte("g"), []byte("val-g"))
batch.Set([]byte("h"), []byte("val-h"))
batch.Set([]byte("i"), []byte("val-i"))
batch.Set([]byte("j"), []byte("val-j"))
err = batch.Execute()
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
it := reader.Iterator([]byte("b"))
key, val, valid := it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "b" {
t.Fatalf("expected key b, got %s", key)
}
if string(val) != "val-b" {
t.Fatalf("expected value val-b, got %s", val)
}
it.Next()
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "c" {
t.Fatalf("expected key c, got %s", key)
}
if string(val) != "val-c" {
t.Fatalf("expected value val-c, got %s", val)
}
it.Seek([]byte("i"))
key, val, valid = it.Current()
if !valid {
t.Fatalf("valid false, expected true")
}
if string(key) != "i" {
t.Fatalf("expected key i, got %s", key)
}
if string(val) != "val-i" {
t.Fatalf("expected value val-i, got %s", val)
}
err = it.Close()
if err != nil {
t.Fatal(err)
}
}
func CommonTestReaderIsolation(t *testing.T, s store.KVStore) {
// insert a kv pair
writer, err := s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("a"), []byte("val-a"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// create an isolated reader
reader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := reader.Close()
if err != nil {
t.Fatal(err)
}
}()
// verify that we see the value already inserted
val, err := reader.Get([]byte("a"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-a")) {
t.Errorf("expected val-a, got nil")
}
// verify that an iterator sees it
count := 0
it := reader.Iterator([]byte{0})
defer func() {
err := it.Close()
if err != nil {
t.Fatal(err)
}
}()
for it.Valid() {
it.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
// add something after the reader was created
writer, err = s.Writer()
if err != nil {
t.Error(err)
}
err = writer.Set([]byte("b"), []byte("val-b"))
if err != nil {
t.Fatal(err)
}
err = writer.Close()
if err != nil {
t.Fatal(err)
}
// ensure that a newer reader sees it
newReader, err := s.Reader()
if err != nil {
t.Error(err)
}
defer func() {
err := newReader.Close()
if err != nil {
t.Fatal(err)
}
}()
val, err = newReader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(val, []byte("val-b")) {
t.Errorf("expected val-b, got nil")
}
// ensure that the director iterator sees it
count = 0
it2 := newReader.Iterator([]byte{0})
defer func() {
err := it2.Close()
if err != nil {
t.Fatal(err)
}
}()
for it2.Valid() {
it2.Next()
count++
}
if count != 2 {
t.Errorf("expected iterator to see 2, saw %d", count)
}
// but that the isolated reader does not
val, err = reader.Get([]byte("b"))
if err != nil {
t.Error(err)
}
if val != nil {
t.Errorf("expected nil, got %v", val)
}
// and ensure that the iterator on the isolated reader also does not
count = 0
it3 := reader.Iterator([]byte{0})
defer func() {
err := it3.Close()
if err != nil {
t.Fatal(err)
}
}()
for it3.Valid() {
it3.Next()
count++
}
if count != 1 {
t.Errorf("expected iterator to see 1, saw %d", count)
}
}

View file

@ -1,50 +1,95 @@
// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package boltdb
import (
"fmt"
"github.com/blevesearch/bleve/index/store"
"github.com/boltdb/bolt"
)
type Writer struct {
store *Store
tx *bolt.Tx
reader *Reader
}
func (w *Writer) Set(key, val []byte) error {
return w.tx.Bucket([]byte(w.store.bucket)).Put(key, val)
}
func (w *Writer) Delete(key []byte) error {
return w.tx.Bucket([]byte(w.store.bucket)).Delete(key)
store *Store
}
func (w *Writer) NewBatch() store.KVBatch {
return store.NewEmulatedBatch(w, w.store.mo)
return store.NewEmulatedBatch(w.store.mo)
}
func (w *Writer) NewBatchEx(options store.KVBatchOptions) ([]byte, store.KVBatch, error) {
return make([]byte, options.TotalBytes), w.NewBatch(), nil
}
func (w *Writer) ExecuteBatch(batch store.KVBatch) (err error) {
emulatedBatch, ok := batch.(*store.EmulatedBatch)
if !ok {
return fmt.Errorf("wrong type of batch")
}
tx, err := w.store.db.Begin(true)
if err != nil {
return
}
// defer function to ensure that once started,
// we either Commit tx or Rollback
defer func() {
// if nothing went wrong, commit
if err == nil {
// careful to catch error here too
err = tx.Commit()
} else {
// caller should see error that caused abort,
// not success or failure of Rollback itself
_ = tx.Rollback()
}
}()
bucket := tx.Bucket([]byte(w.store.bucket))
bucket.FillPercent = w.store.fillPercent
for k, mergeOps := range emulatedBatch.Merger.Merges {
kb := []byte(k)
existingVal := bucket.Get(kb)
mergedVal, fullMergeOk := w.store.mo.FullMerge(kb, existingVal, mergeOps)
if !fullMergeOk {
err = fmt.Errorf("merge operator returned failure")
return
}
err = bucket.Put(kb, mergedVal)
if err != nil {
return
}
}
for _, op := range emulatedBatch.Ops {
if op.V != nil {
err = bucket.Put(op.K, op.V)
if err != nil {
return
}
} else {
err = bucket.Delete(op.K)
if err != nil {
return
}
}
}
return
}
func (w *Writer) Close() error {
w.store.writer.Unlock()
return w.tx.Commit()
}
func (w *Writer) BytesSafeAfterClose() bool {
return w.reader.BytesSafeAfterClose()
}
func (w *Writer) Get(key []byte) ([]byte, error) {
return w.reader.Get(key)
}
func (w *Writer) Iterator(key []byte) store.KVIterator {
return w.reader.Iterator(key)
return nil
}